try:<EOL><INDENT>if "<STR_LIT>" in PyFunceble.INTERN:  <EOL><INDENT>if not Check().is_ip_valid():<EOL><INDENT>request = PyFunceble.socket.getaddrinfo(<EOL>PyFunceble.INTERN["<STR_LIT>"],<EOL><NUM_LIT>,<EOL><NUM_LIT:0>,<EOL><NUM_LIT:0>,<EOL>PyFunceble.socket.IPPROTO_TCP,<EOL>)<EOL>for sequence in request:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"].append(<EOL>sequence[-<NUM_LIT:1>][<NUM_LIT:0>]<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>request = PyFunceble.socket.gethostbyaddr(<EOL>PyFunceble.INTERN["<STR_LIT>"]<EOL>)<EOL>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"][<EOL>"<STR_LIT>"<EOL>] = request[<NUM_LIT:0>]<EOL>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"][<EOL>"<STR_LIT>"<EOL>] = request[<NUM_LIT:1>]<EOL>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"] = request[<EOL><NUM_LIT:2><EOL>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not Check().is_ip_valid():<EOL><INDENT>PyFunceble.socket.getaddrinfo(<EOL>PyFunceble.INTERN["<STR_LIT>"],<EOL><NUM_LIT>,<EOL><NUM_LIT:0>,<EOL><NUM_LIT:0>,<EOL>PyFunceble.socket.IPPROTO_TCP,<EOL>)<EOL><DEDENT>else:<EOL><INDENT>PyFunceble.socket.gethostbyaddr(PyFunceble.INTERN["<STR_LIT>"])<EOL><DEDENT><DEDENT>return True<EOL><DEDENT>except (OSError, PyFunceble.socket.herror, PyFunceble.socket.gaierror):<EOL><INDENT>return False<EOL><DEDENT>


def is_url_valid(self, url=None, return_base=False, return_formatted=False):

<EOL>initial_base = None<EOL>if url:<EOL><INDENT>to_test = url<EOL><DEDENT>elif self.element:<EOL><INDENT>to_test = self.element<EOL><DEDENT>else:<EOL><INDENT>to_test = PyFunceble.INTERN["<STR_LIT>"]<EOL><DEDENT>if to_test.startswith("<STR_LIT:http>"):<EOL><INDENT>try:<EOL><INDENT>regex = r"<STR_LIT>"<EOL>initial_base = base = Regex(<EOL>to_test, regex, return_data=True, rematch=True<EOL>).match()[<NUM_LIT:2>]<EOL>if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>base = domain2idna(base)<EOL><DEDENT>domain_status = self.is_domain_valid(base)<EOL>ip_status = self.is_ip_valid(base)<EOL>if domain_status or ip_status:<EOL><INDENT>if PyFunceble.CONFIGURATION["<STR_LIT>"] and return_formatted:<EOL><INDENT>return Regex(<EOL>to_test,<EOL>initial_base,<EOL>escape=True,<EOL>return_data=True,<EOL>replace_with=base,<EOL>occurences=<NUM_LIT:1>,<EOL>).replace()<EOL><DEDENT>if return_formatted:<EOL><INDENT>return to_test<EOL><DEDENT>if return_base:<EOL><INDENT>return base<EOL><DEDENT>return True<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>if return_formatted:<EOL><INDENT>return to_test<EOL><DEDENT>return False<EOL>

Check if the given URL is valid.

:param url: The url to validate.
:type url: str

:param return_base:
    Allow us the return of the url base (if URL formatted correctly).
:type return_formatted: bool

:param return_formatted:
    Allow us to get the URL converted to IDNA if the conversion
    is activated.
:type return_formatted: bool

:return: The validity of the URL or its base.
:rtype: bool|str


with Popen(self.command, stdout=PIPE, shell=True) as process:<EOL><INDENT>while True:<EOL><INDENT>current_line = process.stdout.readline().rstrip()<EOL>if not current_line:<EOL><INDENT>break<EOL><DEDENT>yield self._decode_output(current_line)<EOL><DEDENT><DEDENT>

Run the given command and yield each line(s) one by one.

.. note::
    The difference between this method and :code:`self.execute()`
    is that :code:`self.execute()` wait for the process to end
    in order to return its output.


try:<EOL><INDENT>with open(destination, "<STR_LIT:w>") as file:<EOL><INDENT>dump(<EOL>self.main_dictionnary,<EOL>file,<EOL>ensure_ascii=False,<EOL>indent=<NUM_LIT:4>,<EOL>sort_keys=True,<EOL>)<EOL><DEDENT><DEDENT>except UnicodeEncodeError:  <EOL><INDENT>with open(destination, "<STR_LIT:w>", encoding="<STR_LIT:utf-8>") as file:<EOL><INDENT>dump(<EOL>self.main_dictionnary,<EOL>file,<EOL>ensure_ascii=False,<EOL>indent=<NUM_LIT:4>,<EOL>sort_keys=True,<EOL>)<EOL><DEDENT><DEDENT>

Save a dictionnary into a JSON file.

:param destination:
    A path to a file where we're going to
    write the converted dict into a JSON format.
:type destination: str


<EOL>result = {}<EOL>if self.algorithm in self.valid_algorithms:<EOL><INDENT>if self.algorithm == "<STR_LIT:all>":<EOL><INDENT>del self.valid_algorithms[<NUM_LIT:0>]<EOL>for algo in self.valid_algorithms:<EOL><INDENT>if self.path and path.isfile(self.path):<EOL><INDENT>result[algo] = self._hash_file(algo)<EOL><DEDENT>elif self.data:<EOL><INDENT>result[algo] = self._hash_data(algo)<EOL><DEDENT>else:  <EOL><INDENT>return None<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if self.path and path.isfile(self.path):<EOL><INDENT>result[self.algorithm] = self._hash_file(self.algorithm)<EOL><DEDENT>elif self.data:<EOL><INDENT>result[self.algorithm] = self._hash_data(self.algorithm)<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT><DEDENT>else:  <EOL><INDENT>return None<EOL><DEDENT>if self.algorithm != "<STR_LIT:all>" and self.only_hash:<EOL><INDENT>return result[self.algorithm]<EOL><DEDENT>return result<EOL>


try:<EOL><INDENT>return loads(data)<EOL><DEDENT>except decoder.JSONDecodeError:  <EOL><INDENT>return {}<EOL><DEDENT>

Convert a JSON formatted string into a dictionary.

:param data: A JSON formatted string to convert to dict format.
:type data: str

:return: The dict representation of the JSON formatted string.
:rtype: dict


pre_result = comp(self.regex)<EOL>return [x for x in self.data if not pre_result.search(str(x))]<EOL>

Return a list of string which don't match the
given regex.


<EOL>process = Popen(self.command, stdout=PIPE, stderr=PIPE, shell=True)<EOL>(output, error) = process.communicate()<EOL>if process.returncode != <NUM_LIT:0>:  <EOL><INDENT>return self._decode_output(error)<EOL><DEDENT>return self._decode_output(output)<EOL>

Execute the given command.

:return: The output of the command.
:rtype: str


if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>to_write = {<EOL>self.current_time: {<EOL>"<STR_LIT>": PyFunceble.INTERN["<STR_LIT>"],<EOL>"<STR_LIT>": extension,<EOL>}<EOL>}<EOL>if self.output:<EOL><INDENT>output = self.output<EOL><DEDENT>else:<EOL><INDENT>output = PyFunceble.OUTPUT_DIRECTORY<EOL>output += PyFunceble.OUTPUTS["<STR_LIT>"]<EOL>output += PyFunceble.OUTPUTS["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"]<EOL>output += PyFunceble.OUTPUTS["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"]<EOL><DEDENT>current_content = self._get_content(output)<EOL>current_content.update(to_write)<EOL>self._write_content(current_content, output)<EOL>if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>PyFunceble.requests.post(<EOL>PyFunceble.LINKS["<STR_LIT>"], data=to_write[self.current_time]<EOL>)<EOL><DEDENT><DEDENT>

Logs the case that the referer was not found.

:param extension: The extension of the domain we are testing.
:type extension: str


if PyFunceble.CONFIGURATION["<STR_LIT>"] and self.backup_content:<EOL><INDENT>file_to_restore = PyFunceble.INTERN["<STR_LIT>"]<EOL>if file_to_restore in self.backup_content:<EOL><INDENT>to_initiate = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>alternatives = {<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>"<STR_LIT>": "<STR_LIT>",<EOL>}<EOL>for string in to_initiate:<EOL><INDENT>try:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"].update(<EOL>{string: self.backup_content[file_to_restore][string]}<EOL>)<EOL><DEDENT>except KeyError:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"].update(<EOL>{<EOL>string: self.backup_content[file_to_restore][<EOL>alternatives[string]<EOL>]<EOL>}<EOL>)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>data_to_backup = {}<EOL>configuration_counter = PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"]<EOL>data_to_backup[PyFunceble.INTERN["<STR_LIT>"]] = {<EOL>"<STR_LIT>": configuration_counter["<STR_LIT>"],<EOL>"<STR_LIT>": configuration_counter["<STR_LIT>"],<EOL>"<STR_LIT>": configuration_counter["<STR_LIT>"],<EOL>"<STR_LIT>": configuration_counter["<STR_LIT>"],<EOL>}<EOL>to_save = {}<EOL>to_save.update(self.backup_content)<EOL>to_save.update(data_to_backup)<EOL>Dict(to_save).to_json(self.autocontinue_log_file)<EOL><DEDENT>


if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>if "<STR_LIT>" not in PyFunceble.INTERN:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = {}<EOL><DEDENT>if PyFunceble.path.isfile(self.file):<EOL><INDENT>data = Dict().from_json(File(self.file).read())<EOL>for file_path in data:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"][file_path] = {}<EOL>for element in data[file_path]:<EOL><INDENT>if data[file_path][element]:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"][file_path][element] = data[<EOL>file_path<EOL>][element]<EOL><DEDENT><DEDENT><DEDENT>return<EOL><DEDENT><DEDENT>PyFunceble.INTERN["<STR_LIT>"] = {}<EOL>return<EOL>


<EOL>list_to_test = self._file_list_to_test_filtering()<EOL>not_filtered = list_to_test<EOL>try:<EOL><INDENT>list_to_test = List(<EOL>list(<EOL>set(<EOL>list_to_test[PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"] :]<EOL>)<EOL>- set(PyFunceble.INTERN["<STR_LIT>"])<EOL>)<EOL>).format()<EOL>_ = list_to_test[-<NUM_LIT:1>]<EOL><DEDENT>except IndexError:<EOL><INDENT>list_to_test = not_filtered[<EOL>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"] :<EOL>]<EOL>del not_filtered<EOL><DEDENT>if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)<EOL><DEDENT>try:<EOL><INDENT>return [self.url(x, list_to_test[-<NUM_LIT:1>]) for x in list_to_test if x]<EOL><DEDENT>except IndexError:<EOL><INDENT>print(PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "<STR_LIT>")<EOL><DEDENT>

Manage the case that we have to test a file

.. note::
    1 URL per line.


if (<EOL>not PyFunceble.CONFIGURATION["<STR_LIT>"]<EOL>and not PyFunceble.CONFIGURATION["<STR_LIT>"]<EOL>):<EOL><INDENT>print("<STR_LIT:\n>")<EOL>if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>Prints(None, "<STR_LIT>").header()<EOL><DEDENT>else:<EOL><INDENT>Prints(None, "<STR_LIT>").header()<EOL><DEDENT>PyFunceble.CONFIGURATION["<STR_LIT>"] = True<EOL><DEDENT>


if not self.modulo_test:  <EOL><INDENT>PyFunceble.INTERN[<EOL>"<STR_LIT>"<EOL>] = self.file_path  <EOL>self._entry_management_url()<EOL>AutoSave().travis_permissions()<EOL>self.bypass()<EOL>ExecutionTime("<STR_LIT:start>")<EOL>if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>PyFunceble.HTTP_CODE["<STR_LIT>"] = False<EOL><DEDENT>if self.domain_or_ip_to_test:  <EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = "<STR_LIT>"<EOL>ExecutionTime("<STR_LIT:start>")<EOL>PyFunceble.CONFIGURATION["<STR_LIT>"] = False<EOL>PyFunceble.CONFIGURATION["<STR_LIT>"] = False<EOL>if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>domain_or_ip_to_test = domain2idna(<EOL>self.domain_or_ip_to_test.lower()  <EOL>)<EOL><DEDENT>else:<EOL><INDENT>domain_or_ip_to_test = (<EOL>self.domain_or_ip_to_test.lower()  <EOL>)  <EOL><DEDENT>self.domain(domain_or_ip_to_test)<EOL><DEDENT>elif self.url_to_test and not self.file_path:  <EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = "<STR_LIT:url>"<EOL>ExecutionTime("<STR_LIT:start>")<EOL>PyFunceble.CONFIGURATION["<STR_LIT>"] = False<EOL>self.url(<EOL>self.checker.is_url_valid(<EOL>self.url_to_test,  <EOL>return_formatted=True,<EOL>)<EOL>)<EOL><DEDENT>elif (<EOL>self._entry_management_url_download(<EOL>self.url_file  <EOL>)<EOL>or self.url_file  <EOL>):<EOL><INDENT>PyFunceble.CONFIGURATION["<STR_LIT>"] = PyFunceble.CONFIGURATION[<EOL>"<STR_LIT>"<EOL>] = PyFunceble.CONFIGURATION["<STR_LIT>"] = True<EOL>PyFunceble.CONFIGURATION["<STR_LIT>"] = False<EOL>PyFunceble.INTERN["<STR_LIT>"] = "<STR_LIT:url>"<EOL>self.file_url()<EOL><DEDENT>elif (<EOL>self._entry_management_url_download(<EOL>self.link_to_test  <EOL>)<EOL>or self._entry_management_url_download(<EOL>self.file_path  <EOL>)  <EOL>or self.file_path  <EOL>):<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = "<STR_LIT>"<EOL>self.file()<EOL><DEDENT>else:<EOL><INDENT>print(<EOL>PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "<STR_LIT>"<EOL>)<EOL><DEDENT>if (<EOL>self.domain_or_ip_to_test  <EOL>or self.url_to_test  <EOL>):<EOL><INDENT>ExecutionTime("<STR_LIT>", last=True)<EOL>self.percentage.log()<EOL>self.colorify_logo()<EOL><DEDENT>PyFunceble.stay_safe()<EOL><DEDENT>else:<EOL><INDENT>PyFunceble.CONFIGURATION["<STR_LIT>"] = PyFunceble.CONFIGURATION[<EOL>"<STR_LIT>"<EOL>] = PyFunceble.CONFIGURATION["<STR_LIT>"] = True<EOL>PyFunceble.CONFIGURATION["<STR_LIT>"] = PyFunceble.CONFIGURATION[<EOL>"<STR_LIT>"<EOL>] = PyFunceble.CONFIGURATION["<STR_LIT>"] = PyFunceble.CONFIGURATION[<EOL>"<STR_LIT>"<EOL>] = False<EOL>if self.domain_or_ip_to_test:  <EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = "<STR_LIT>"<EOL>PyFunceble.INTERN[<EOL>"<STR_LIT>"<EOL>] = self.domain_or_ip_to_test.lower()  <EOL><DEDENT>elif self.url_to_test:  <EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = "<STR_LIT:url>"<EOL>PyFunceble.INTERN[<EOL>"<STR_LIT>"<EOL>] = self.url_to_test<EOL><DEDENT><DEDENT>

Avoid to have 1 millions line into self.__init__()


if not PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>to_print = []<EOL>if home:<EOL><INDENT>for line in PyFunceble.ASCII_PYFUNCEBLE.split("<STR_LIT:\n>"):<EOL><INDENT>to_print.append(<EOL>PyFunceble.Fore.YELLOW + line + PyFunceble.Fore.RESET<EOL>)<EOL><DEDENT><DEDENT>elif PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"]["<STR_LIT>"] >= <NUM_LIT:50>:<EOL><INDENT>for line in PyFunceble.ASCII_PYFUNCEBLE.split("<STR_LIT:\n>"):<EOL><INDENT>to_print.append(<EOL>PyFunceble.Fore.GREEN + line + PyFunceble.Fore.RESET<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for line in PyFunceble.ASCII_PYFUNCEBLE.split("<STR_LIT:\n>"):<EOL><INDENT>to_print.append(PyFunceble.Fore.RED + line + PyFunceble.Fore.RESET)<EOL><DEDENT><DEDENT>print("<STR_LIT:\n>".join(to_print))<EOL><DEDENT>

Print the colored logo based on global results.

:param home: Tell us if we have to print the initial coloration.
:type home: bool


if (<EOL>self.url_file  <EOL>and not self._entry_management_url_download(<EOL>self.url_file  <EOL>)<EOL>):  <EOL><INDENT>PyFunceble.INTERN[<EOL>"<STR_LIT>"<EOL>] = self.url_file<EOL><DEDENT>


for string in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"]["<STR_LIT>"].update({string: <NUM_LIT:0>})<EOL><DEDENT>


return Dict().from_yaml(<EOL>File(PyFunceble.CURRENT_DIRECTORY + "<STR_LIT>").read()<EOL>)<EOL>

Get and return the content of version.yaml


to_ignore = ["<STR_LIT>", "<STR_LIT>"]<EOL>for root, _, files in PyFunceble.walk(<EOL>PyFunceble.CURRENT_DIRECTORY<EOL>+ PyFunceble.directory_separator<EOL>+ "<STR_LIT>"<EOL>+ PyFunceble.directory_separator<EOL>):<EOL><INDENT>for file in files:<EOL><INDENT>if file not in to_ignore and "<STR_LIT>" not in root:<EOL><INDENT>if root.endswith(PyFunceble.directory_separator):<EOL><INDENT>self._update_docs(root + file)<EOL><DEDENT>else:<EOL><INDENT>self._update_docs(root + PyFunceble.directory_separator + file)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for root, _, files in PyFunceble.walk(<EOL>PyFunceble.CURRENT_DIRECTORY<EOL>+ PyFunceble.directory_separator<EOL>+ "<STR_LIT>"<EOL>+ PyFunceble.directory_separator<EOL>):<EOL><INDENT>for file in files:<EOL><INDENT>if file not in to_ignore and "<STR_LIT>" not in root:<EOL><INDENT>if root.endswith(PyFunceble.directory_separator):<EOL><INDENT>self._update_docs(root + file)<EOL><DEDENT>else:<EOL><INDENT>self._update_docs(root + PyFunceble.directory_separator + file)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>


if (<EOL>self._authorization()<EOL>and self.is_in_database()<EOL>and int(<EOL>PyFunceble.INTERN["<STR_LIT>"][PyFunceble.INTERN["<STR_LIT>"]][<EOL>PyFunceble.INTERN["<STR_LIT>"]<EOL>]["<STR_LIT>"]<EOL>)<EOL>< int(PyFunceble.time())<EOL>):<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

Check if the current time is older than the one in the database.


if self._authorization():<EOL><INDENT>Dict(PyFunceble.INTERN["<STR_LIT>"]).to_json(self.whois_db_path)<EOL><DEDENT>


if self._authorization() and self.is_in_database() and not self.is_time_older():<EOL><INDENT>result = PyFunceble.INTERN["<STR_LIT>"][PyFunceble.INTERN["<STR_LIT>"]][<EOL>PyFunceble.INTERN["<STR_LIT>"]<EOL>]["<STR_LIT>"]<EOL>if result:<EOL><INDENT>return result<EOL><DEDENT><DEDENT>return None<EOL>

Get the expiration date from the database.

:return: The expiration date from the database.
:rtype: str|None


if (<EOL>not PyFunceble.CONFIGURATION["<STR_LIT>"]<EOL>and PyFunceble.CONFIGURATION["<STR_LIT>"]<EOL>):<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

Check if we are authorized to work with our database.


if PyFunceble.CONFIGURATION["<STR_LIT>"]:<EOL><INDENT>Dict(PyFunceble.INTERN["<STR_LIT>"]).to_json(self.inactive_db_path)<EOL><DEDENT>

Save the current database into the inactive-db.json file.


<EOL>short_month = {<EOL>"<STR_LIT>": [str(<NUM_LIT:1>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:2>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:3>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:4>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:5>), "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:6>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:7>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:8>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:9>), "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:10>), "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:11>), "<STR_LIT>", "<STR_LIT>"],<EOL>"<STR_LIT>": [str(<NUM_LIT:12>), "<STR_LIT>", "<STR_LIT>"],<EOL>}<EOL>for month in short_month:<EOL><INDENT>if data in short_month[month]:<EOL><INDENT>return month<EOL><DEDENT><DEDENT>return data<EOL>

Convert a given month into our unified format.

:param data: The month to convert or shorten.
:type data: str

:return: The unified month name.
:rtype: str


<EOL>domain_validation = self.checker.is_domain_valid()<EOL>ip_validation = self.checker.is_ip_valid()<EOL>if "<STR_LIT>" in PyFunceble.INTERN:<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"].update(<EOL>{<EOL>"<STR_LIT>": domain_validation,<EOL>"<STR_LIT>": ip_validation,<EOL>}<EOL>)<EOL><DEDENT>if (<EOL>domain_validation<EOL>and not ip_validation<EOL>or domain_validation<EOL>or PyFunceble.CONFIGURATION["<STR_LIT>"]<EOL>):<EOL><INDENT>PyFunceble.INTERN.update(<EOL>{"<STR_LIT>": HTTPCode().get(), "<STR_LIT>": Referer().get()}<EOL>)<EOL>if not PyFunceble.INTERN["<STR_LIT>"]:<EOL><INDENT>return PyFunceble.INTERN["<STR_LIT>"]<EOL><DEDENT>if PyFunceble.INTERN["<STR_LIT>"] and not self.checker.is_subdomain():<EOL><INDENT>return self._extract()<EOL><DEDENT>Logs().whois(self.whois_record)<EOL>return None<EOL><DEDENT>if (<EOL>ip_validation<EOL>and not domain_validation<EOL>or ip_validation<EOL>or PyFunceble.CONFIGURATION["<STR_LIT>"]<EOL>):<EOL><INDENT>PyFunceble.INTERN["<STR_LIT>"] = HTTPCode().get()<EOL>Logs().whois(self.whois_record)<EOL>return None<EOL><DEDENT>Logs().whois(self.whois_record)<EOL>return False<EOL>

Execute the logic behind the meaning of ExpirationDate + return the matched status.

:return:
    The status of the tested domain.
    Can be one of the official status.
:rtype: str


@classmethod<EOL><INDENT>def handle(cls, status, invalid_source="<STR_LIT>"):<DEDENT>

if status.lower() not in PyFunceble.STATUS["<STR_LIT:list>"]["<STR_LIT>"]:<EOL><INDENT>source = "<STR_LIT>"<EOL>if Lookup().nslookup():<EOL><INDENT>status, source = cls.extra_rules.handle(<EOL>PyFunceble.STATUS["<STR_LIT>"]["<STR_LIT>"], source<EOL>)<EOL>Generate(status, source).status_file()<EOL>return status, source<EOL><DEDENT>status, source = cls.extra_rules.handle(<EOL>PyFunceble.STATUS["<STR_LIT>"]["<STR_LIT>"], source<EOL>)<EOL>Generate(status, source).status_file()<EOL>return status, source<EOL><DEDENT>status, source = cls.extra_rules.handle(<EOL>PyFunceble.STATUS["<STR_LIT>"]["<STR_LIT>"], invalid_source<EOL>)<EOL>Generate(status, source).status_file()<EOL>return status, source<EOL>

Handle the lack of WHOIS and expiration date. :smile_cat:

:param matched_status: The status that we have to handle.
:type status: str

:param invalid_source:
    The source to set when we handle INVALID element.
:type invalid_source: str

:return:
    The strus of the domain after generating the files desired
    by the user.
:rtype: str


return self.storage_path.rsplit('<STR_LIT:/>', <NUM_LIT:1>)[<NUM_LIT:1>]<EOL>


client = self.get_client()<EOL>yield client<EOL>self.put_client(client)<EOL>

A context manager that obtains a client for use, whether an
existing unused client or a brand new one if none are
available.


pass<EOL>

Resets the client, closing any connections and discarding any
state. This can be useful if some exceptional condition
occurred and the request/response state can no longer be
certain.


def post_account(self, headers=None, query=None, cdn=False, body=None):

return self.request(<EOL>'<STR_LIT:POST>', '<STR_LIT>', body or '<STR_LIT>', headers, query=query, cdn=cdn)<EOL>

POSTs the account and returns the results. This is usually
done to set X-Account-Meta-xxx headers. Note that any existing
X-Account-Meta-xxx headers will remain untouched. To remove an
X-Account-Meta-xxx header, send the header with an empty
string as its value.

:param headers: Additional headers to send with the request.
:param query: Set to a dict of query values to send on the
    query string of the request.
:param cdn: If set True, the CDN management interface will be
    used.
:param body: No known Swift POSTs take a body; but the option
    is there for the future.
:returns: A tuple of (status, reason, headers, contents).

    :status: is an int for the HTTP status code.
    :reason: is the str for the HTTP status (ex: "Ok").
    :headers: is a dict with all lowercase keys of the HTTP
        headers; if a header has multiple values, it will be a
        list.
    :contents: is the str for the HTTP body.


def put_container(self, container, headers=None, query=None, cdn=False,<EOL>body=None):

path = self._container_path(container)<EOL>return self.request(<EOL>'<STR_LIT>', path, body or '<STR_LIT>', headers, query=query, cdn=cdn)<EOL>

PUTs the container and returns the results. This is usually
done to create new containers and can also be used to set
X-Container-Meta-xxx headers. Note that if the container
already exists, any existing X-Container-Meta-xxx headers will
remain untouched. To remove an X-Container-Meta-xxx header,
send the header with an empty string as its value.

:param container: The name of the container.
:param headers: Additional headers to send with the request.
:param query: Set to a dict of query values to send on the
    query string of the request.
:param cdn: If set True, the CDN management interface will be
    used.
:param body: Some container PUT requests, like the
    extract-archive bulk upload request, take a body.
:returns: A tuple of (status, reason, headers, contents).

    :status: is an int for the HTTP status code.
    :reason: is the str for the HTTP status (ex: "Ok").
    :headers: is a dict with all lowercase keys of the HTTP
        headers; if a header has multiple values, it will be a
        list.
    :contents: is the str for the HTTP body.


def get_account(self, headers=None, prefix=None, delimiter=None,<EOL>marker=None, end_marker=None, limit=None, query=None,<EOL>cdn=False, decode_json=True):

query = dict(query or {})<EOL>query['<STR_LIT>'] = '<STR_LIT>'<EOL>if prefix:<EOL><INDENT>query['<STR_LIT>'] = prefix<EOL><DEDENT>if delimiter:<EOL><INDENT>query['<STR_LIT>'] = delimiter<EOL><DEDENT>if marker:<EOL><INDENT>query['<STR_LIT>'] = marker<EOL><DEDENT>if end_marker:<EOL><INDENT>query['<STR_LIT>'] = end_marker<EOL><DEDENT>if limit:<EOL><INDENT>query['<STR_LIT>'] = limit<EOL><DEDENT>return self.request(<EOL>'<STR_LIT:GET>', '<STR_LIT>', '<STR_LIT>', headers, decode_json=decode_json, query=query,<EOL>cdn=cdn)<EOL>

GETs the account and returns the results. This is done to list
the containers for the account. Some useful headers are also
returned:

=========================== =================================
x-account-bytes-used        Object storage used for the
                            account, in bytes.
x-account-container-count   The number of containers in the
                            account.
x-account-object-count      The number of objects in the
                            account.
=========================== =================================

Also, any user headers beginning with x-account-meta- are
returned.

These values can be delayed depending the Swift cluster.

:param headers: Additional headers to send with the request.
:param prefix: The prefix container names must match to be
    listed.
:param delimiter: The delimiter for the listing. Delimiters
    indicate how far to progress through container names
    before "rolling them up". For instance, a delimiter='.'
    query on an account with the containers::

        one.one
        one.two
        two
        three.one

    would return the JSON value of::

        [{'subdir': 'one.'},
         {'count': 0, 'bytes': 0, 'name': 'two'},
         {'subdir': 'three.'}]

    Using this with prefix can allow you to traverse a psuedo
    hierarchy.
:param marker: Only container names after this marker will be
    returned. Swift returns a limited number of containers
    per request (often 10,000). To get the next batch of
    names, you issue another query with the marker set to the
    last name you received. You can continue to issue
    requests until you receive no more names.
:param end_marker: Only container names before this marker will be
    returned.
:param limit: Limits the size of the list returned per
    request. The default and maximum depends on the Swift
    cluster (usually 10,000).
:param query: Set to a dict of query values to send on the
    query string of the request.
:param cdn: If set True, the CDN management interface will be
    used.
:param decode_json: If set False, the usual decoding of the
    JSON response will be skipped and the raw contents will
    be returned instead.
:returns: A tuple of (status, reason, headers, contents).

    :status: is an int for the HTTP status code.
    :reason: is the str for the HTTP status (ex: "Ok").
    :headers: is a dict with all lowercase keys of the HTTP
        headers; if a header has multiple values, it will be a
        list.
    :contents: is the decoded JSON response or the raw str
        for the HTTP body.


def get_object(self, container, obj, headers=None, stream=True, query=None,<EOL>cdn=False):

path = self._object_path(container, obj)<EOL>return self.request(<EOL>'<STR_LIT:GET>', path, '<STR_LIT>', headers, query=query, stream=stream, cdn=cdn)<EOL>

GETs the object and returns the results.

:param container: The name of the container.
:param obj: The name of the object.
:param headers: Additional headers to send with the request.
:param stream: Indicates whether to stream the contents or
    preread them fully and return them as a str. Default:
    True to stream the contents. When streaming, contents
    will have the standard file-like-object read function,
    which accepts an optional size parameter to limit how
    much data is read per call. When streaming is on, be
    certain to fully read the contents before issuing another
    request.
:param query: Set to a dict of query values to send on the
    query string of the request.
:param cdn: If set True, the CDN management interface will be
    used.
:returns: A tuple of (status, reason, headers, contents).

    :status: is an int for the HTTP status code.
    :reason: is the str for the HTTP status (ex: "Ok").
    :headers: is a dict with all lowercase keys of the HTTP
        headers; if a header has multiple values, it will be a
        list.
    :contents: if *stream* was True, *contents* is a
        file-like-object of the contents of the HTTP body. If
        *stream* was False, *contents* is just a simple str of
        the HTTP body.


def delete_account(self, headers=None,<EOL>yes_i_mean_delete_the_account=False, query=None,<EOL>cdn=False, body=None):

if not yes_i_mean_delete_the_account and (<EOL>not body or not query or '<STR_LIT>' not in query):<EOL><INDENT>return (<NUM_LIT:0>, '<STR_LIT>', {},<EOL>'<STR_LIT>')<EOL><DEDENT>return self.request(<EOL>'<STR_LIT>', '<STR_LIT>', body or '<STR_LIT>', headers, query=query, cdn=cdn)<EOL>

Sends a DELETE request to the account and returns the results.

With ``query['bulk-delete'] = ''`` this might mean a bulk
delete request where the body of the request is new-line
separated, url-encoded list of names to delete. Be careful
with this! One wrong move and you might mark your account for
deletion of you have the access to do so!

For a plain DELETE to the account, on clusters that support
it and, assuming you have permissions to do so, the account
will be marked as deleted and immediately begin removing the
objects from the cluster in the backgound.

THERE IS NO GOING BACK!

:param headers: Additional headers to send with the request.
:param yes_i_mean_delete_the_account: Set to True to verify
    you really mean to delete the entire account. This is
    required unless ``body and 'bulk-delete' in query``.
:param query: Set to a dict of query values to send on the
    query string of the request.
:param cdn: If set True, the CDN management interface will be
    used.
:param body: Some account DELETE requests, like the bulk
    delete request, take a body.
:returns: A tuple of (status, reason, headers, contents).

    :status: is an int for the HTTP status code.
    :reason: is the str for the HTTP status (ex: "Ok").
    :headers: is a dict with all lowercase keys of the HTTP
        headers; if a header has multiple values, it will be
        a list.
    :contents: is the str for the HTTP body.


if isinstance(value, six.text_type):<EOL><INDENT>value = value.encode('<STR_LIT:utf8>')<EOL><DEDENT>elif not isinstance(value, six.string_types):<EOL><INDENT>value = str(value)<EOL><DEDENT>return parse.quote(value, safe)<EOL>

Much like parse.quote in that it returns a URL encoded string
for the given value, protecting the safe characters; but this
version also ensures the value is UTF-8 encoded.


def write_headers(self, fp, headers, mute=None):

if headers:<EOL><INDENT>if not mute:<EOL><INDENT>mute = []<EOL><DEDENT>fmt = '<STR_LIT>' % (max(len(k) for k in headers) + <NUM_LIT:1>)<EOL>for key in sorted(headers):<EOL><INDENT>if key in mute:<EOL><INDENT>continue<EOL><DEDENT>fp.write(fmt % (key.title() + '<STR_LIT::>', headers[key]))<EOL><DEDENT>fp.flush()<EOL><DEDENT>

Convenience function to output headers in a formatted fashion
to a file-like fp, optionally muting any headers in the mute
list.


context = CLIContext()<EOL>for item in dir(self):<EOL><INDENT>if item[<NUM_LIT:0>] != '<STR_LIT:_>' and item not in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>setattr(context, item, getattr(self, item))<EOL><DEDENT><DEDENT>return context<EOL>

Returns a new CLIContext instance that is a shallow copy of
the original, much like dict's copy method.


limit = context.query.get('<STR_LIT>')<EOL>delimiter = context.query.get('<STR_LIT>')<EOL>prefix = context.query.get('<STR_LIT>')<EOL>marker = context.query.get('<STR_LIT>')<EOL>end_marker = context.query.get('<STR_LIT>')<EOL>if context.raw:<EOL><INDENT>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_account(<EOL>decode_json=False, headers=context.headers, limit=limit,<EOL>marker=marker, end_marker=end_marker, query=context.query,<EOL>cdn=context.cdn)<EOL>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents = contents.read()<EOL><DEDENT><DEDENT>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>raise ReturnCode('<STR_LIT>' % (status, reason))<EOL><DEDENT>with context.io_manager.with_stdout() as fp:<EOL><INDENT>if context.output_headers:<EOL><INDENT>context.write_headers(<EOL>fp, headers, context.muted_account_headers)<EOL><DEDENT>fp.write(contents)<EOL>fp.flush()<EOL><DEDENT>return<EOL><DEDENT>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_account(<EOL>headers=context.headers, limit=limit, marker=marker,<EOL>end_marker=end_marker, query=context.query, cdn=context.cdn)<EOL>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents.read()<EOL><DEDENT>raise ReturnCode('<STR_LIT>' % (status, reason))<EOL><DEDENT><DEDENT>if context.output_headers and not context.all_objects:<EOL><INDENT>with context.io_manager.with_stdout() as fp:<EOL><INDENT>context.write_headers(<EOL>fp, headers, context.muted_account_headers)<EOL><DEDENT><DEDENT>while contents:<EOL><INDENT>if context.all_objects:<EOL><INDENT>new_context = context.copy()<EOL>new_context.query = dict(new_context.query)<EOL>for remove in (<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if remove in new_context.query:<EOL><INDENT>del new_context.query[remove]<EOL><DEDENT><DEDENT>for item in contents:<EOL><INDENT>if '<STR_LIT:name>' in item:<EOL><INDENT>new_path = item['<STR_LIT:name>'].encode('<STR_LIT:utf8>')<EOL>cli_get_container_listing(new_context, new_path)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>with context.io_manager.with_stdout() as fp:<EOL><INDENT>for item in contents:<EOL><INDENT>if context.full:<EOL><INDENT>fp.write('<STR_LIT>' % (<EOL>item.get('<STR_LIT>', '<STR_LIT:->'),<EOL>item.get('<STR_LIT:count>', '<STR_LIT:->')))<EOL><DEDENT>fp.write(item.get(<EOL>'<STR_LIT:name>', item.get('<STR_LIT>')))<EOL>fp.write('<STR_LIT:\n>')<EOL><DEDENT>fp.flush()<EOL><DEDENT><DEDENT>if limit:<EOL><INDENT>break<EOL><DEDENT>marker = contents[-<NUM_LIT:1>].get('<STR_LIT:name>', contents[-<NUM_LIT:1>].get('<STR_LIT>', '<STR_LIT>'))<EOL>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_account(<EOL>headers=context.headers, limit=limit, delimiter=delimiter,<EOL>prefix=prefix, end_marker=end_marker, marker=marker,<EOL>query=context.query, cdn=context.cdn)<EOL>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents.read()<EOL><DEDENT>raise ReturnCode('<STR_LIT>' % (status, reason))<EOL><DEDENT><DEDENT><DEDENT>

Performs a GET on the account as a listing request.

See :py:mod:`swiftly.cli.get` for context usage information.

See :py:class:`CLIGet` for more information.


path = path.lstrip('<STR_LIT:/>') if path else None<EOL>if not path:<EOL><INDENT>return cli_get_account_listing(context)<EOL><DEDENT>elif '<STR_LIT:/>' not in path.rstrip('<STR_LIT:/>'):<EOL><INDENT>return cli_get_container_listing(context, path)<EOL><DEDENT>status, reason, headers, contents = <NUM_LIT:0>, '<STR_LIT>', {}, '<STR_LIT>'<EOL>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_object(<EOL>*path.split('<STR_LIT:/>', <NUM_LIT:1>), headers=context.headers, query=context.query,<EOL>cdn=context.cdn)<EOL>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents.read()<EOL><DEDENT>raise ReturnCode(<EOL>'<STR_LIT>' % (path, status, reason))<EOL><DEDENT>if context.decrypt:<EOL><INDENT>crypt_type = contents.read(<NUM_LIT:1>)<EOL>if crypt_type == AES256CBC:<EOL><INDENT>contents = FileLikeIter(aes_decrypt(<EOL>context.decrypt, contents,<EOL>chunk_size=getattr(client, '<STR_LIT>', <NUM_LIT>)))<EOL><DEDENT>else:<EOL><INDENT>raise ReturnCode(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % (path, crypt_type))<EOL><DEDENT><DEDENT>def disk_closed_callback(disk_path):<EOL><INDENT>if context.remove_empty_files and not os.path.getsize(disk_path):<EOL><INDENT>os.unlink(disk_path)<EOL>if context.io_manager.stdout_root:<EOL><INDENT>dirname = os.path.dirname(disk_path)<EOL>while dirname and dirname.startswith(<EOL>context.io_manager.stdout_root):<EOL><INDENT>try:<EOL><INDENT>os.rmdir(dirname)<EOL><DEDENT>except OSError:<EOL><INDENT>pass<EOL><DEDENT>dirname = os.path.dirname(dirname)<EOL><DEDENT><DEDENT>return<EOL><DEDENT>if (headers.get('<STR_LIT>') in<EOL>['<STR_LIT>', '<STR_LIT>'] and<EOL>headers.get('<STR_LIT>') == '<STR_LIT:0>'):<EOL><INDENT>os.unlink(disk_path)<EOL>os.makedirs(disk_path)<EOL><DEDENT>mtime = <NUM_LIT:0><EOL>if '<STR_LIT>' in headers:<EOL><INDENT>mtime = float(headers['<STR_LIT>'])<EOL><DEDENT>elif '<STR_LIT>' in headers:<EOL><INDENT>mtime = time.mktime(time.strptime(<EOL>headers['<STR_LIT>'], '<STR_LIT>'))<EOL><DEDENT>if mtime:<EOL><INDENT>os.utime(disk_path, (mtime, mtime))<EOL><DEDENT><DEDENT>out_path = path<EOL>if context.suppress_container_name:<EOL><INDENT>out_path = out_path.split('<STR_LIT:/>', <NUM_LIT:1>)[<NUM_LIT:1>]<EOL><DEDENT>out_path = context.io_manager.client_path_to_os_path(out_path)<EOL>with context.io_manager.with_stdout(<EOL>out_path, disk_closed_callback=disk_closed_callback) as fp:<EOL><INDENT>if context.output_headers:<EOL><INDENT>context.write_headers(<EOL>fp, headers, context.muted_object_headers)<EOL>fp.write('<STR_LIT:\n>')<EOL><DEDENT>chunk = contents.read(<NUM_LIT>)<EOL>while chunk:<EOL><INDENT>fp.write(chunk)<EOL>chunk = contents.read(<NUM_LIT>)<EOL><DEDENT>fp.flush()<EOL><DEDENT><DEDENT>

Performs a GET on the item (account, container, or object).

See :py:mod:`swiftly.cli.get` for context usage information.

See :py:class:`CLIGet` for more information.


path = path.strip('<STR_LIT:/>') if path else None<EOL>if not path or '<STR_LIT:/>' in path:<EOL><INDENT>raise ReturnCode(<EOL>'<STR_LIT>' %<EOL>path)<EOL><DEDENT>context.suppress_container_name = True<EOL>limit = context.query.get('<STR_LIT>')<EOL>delimiter = context.query.get('<STR_LIT>')<EOL>prefix = context.query.get('<STR_LIT>')<EOL>marker = context.query.get('<STR_LIT>')<EOL>end_marker = context.query.get('<STR_LIT>')<EOL>if context.raw:<EOL><INDENT>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_container(<EOL>path, decode_json=False, headers=context.headers, limit=limit,<EOL>marker=marker, end_marker=end_marker, query=context.query,<EOL>cdn=context.cdn)<EOL>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents = contents.read()<EOL><DEDENT><DEDENT>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>raise ReturnCode(<EOL>'<STR_LIT>' % (path, status, reason))<EOL><DEDENT>with context.io_manager.with_stdout() as fp:<EOL><INDENT>if context.output_headers:<EOL><INDENT>context.write_headers(<EOL>fp, headers, context.muted_container_headers)<EOL><DEDENT>fp.write(contents)<EOL>fp.flush()<EOL><DEDENT>return<EOL><DEDENT>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_container(<EOL>path, headers=context.headers, limit=limit, delimiter=delimiter,<EOL>prefix=prefix, marker=marker, end_marker=end_marker,<EOL>query=context.query, cdn=context.cdn)<EOL>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents.read()<EOL><DEDENT>raise ReturnCode(<EOL>'<STR_LIT>' % (path, status, reason))<EOL><DEDENT><DEDENT>if context.output_headers and not context.all_objects:<EOL><INDENT>with context.io_manager.with_stdout() as fp:<EOL><INDENT>context.write_headers(<EOL>fp, headers, context.muted_container_headers)<EOL><DEDENT><DEDENT>conc = Concurrency(context.concurrency)<EOL>while contents:<EOL><INDENT>if context.all_objects:<EOL><INDENT>new_context = context.copy()<EOL>new_context.query = dict(new_context.query)<EOL>for remove in (<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if remove in new_context.query:<EOL><INDENT>del new_context.query[remove]<EOL><DEDENT><DEDENT>for item in contents:<EOL><INDENT>if '<STR_LIT:name>' in item:<EOL><INDENT>for (exc_type, exc_value, exc_tb, result) insix.itervalues(conc.get_results()):<EOL><INDENT>if exc_value:<EOL><INDENT>conc.join()<EOL>raise exc_value<EOL><DEDENT><DEDENT>new_path = path + '<STR_LIT:/>' + item['<STR_LIT:name>'].encode('<STR_LIT:utf8>')<EOL>conc.spawn(new_path, cli_get, new_context, new_path)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>with context.io_manager.with_stdout() as fp:<EOL><INDENT>for item in contents:<EOL><INDENT>if context.full:<EOL><INDENT>fp.write('<STR_LIT>' % (<EOL>item.get('<STR_LIT>', '<STR_LIT:->'),<EOL>item.get('<STR_LIT>', '<STR_LIT:->')[:<NUM_LIT>].replace(<EOL>'<STR_LIT:T>', '<STR_LIT:U+0020>'),<EOL>item.get('<STR_LIT>', '<STR_LIT:->'),<EOL>item.get('<STR_LIT>', '<STR_LIT:->')))<EOL><DEDENT>fp.write(item.get(<EOL>'<STR_LIT:name>', item.get('<STR_LIT>')))<EOL>fp.write('<STR_LIT:\n>')<EOL><DEDENT>fp.flush()<EOL><DEDENT><DEDENT>if limit:<EOL><INDENT>break<EOL><DEDENT>marker = contents[-<NUM_LIT:1>].get('<STR_LIT:name>', contents[-<NUM_LIT:1>].get('<STR_LIT>', '<STR_LIT>'))<EOL>with context.client_manager.with_client() as client:<EOL><INDENT>status, reason, headers, contents = client.get_container(<EOL>path, headers=context.headers, limit=limit,<EOL>delimiter=delimiter, prefix=prefix, end_marker=end_marker,<EOL>marker=marker, query=context.query, cdn=context.cdn)<EOL>if status // <NUM_LIT:100> != <NUM_LIT:2>:<EOL><INDENT>if status == <NUM_LIT> and context.ignore_404:<EOL><INDENT>return<EOL><DEDENT>if hasattr(contents, '<STR_LIT>'):<EOL><INDENT>contents.read()<EOL><DEDENT>raise ReturnCode(<EOL>'<STR_LIT>' % (path, status, reason))<EOL><DEDENT><DEDENT><DEDENT>conc.join()<EOL>for (exc_type, exc_value, exc_tb, result) insix.itervalues(conc.get_results()):<EOL><INDENT>if exc_value:<EOL><INDENT>raise exc_value<EOL><DEDENT><DEDENT>

Performs a GET on the container as a listing request.

See :py:mod:`swiftly.cli.get` for context usage information.

See :py:class:`CLIGet` for more information.


def cli_help(context, command_name, general_parser, command_parsers):

if command_name == '<STR_LIT>':<EOL><INDENT>command_name = '<STR_LIT>'<EOL><DEDENT>with context.io_manager.with_stdout() as stdout:<EOL><INDENT>if not command_name:<EOL><INDENT>general_parser.print_help(stdout)<EOL><DEDENT>elif command_name in command_parsers:<EOL><INDENT>command_parsers[command_name].option_parser.print_help(stdout)<EOL><DEDENT>else:<EOL><INDENT>raise ReturnCode('<STR_LIT>' % command_name)<EOL><DEDENT><DEDENT>

Outputs help information.

See :py:mod:`swiftly.cli.help` for context usage information.

See :py:class:`CLIHelp` for more information.

:param context: The :py:class:`swiftly.cli.context.CLIContext` to
    use.
:param command_name: The command_name to output help information
    for, or set to None or an empty string to output the general
    help information.
:param general_parser: The
    :py:class:`swiftly.cli.optionparser.OptionParser` for general
    usage.
:param command_parsers: A dict of (name, :py:class:`CLICommand`)
    for specific command usage.


def wrapper(self, msg, file=None):<EOL><INDENT>if file:<EOL><INDENT>return func(self, msg, file=file)<EOL><DEDENT>elif self.io_manager:<EOL><INDENT>with self.io_manager.with_stderr() as stderr:<EOL><INDENT>return func(self, msg, file=stderr)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return func(self, msg, file=sys.stderr)<EOL><DEDENT><DEDENT>wrapper.__doc__ = func.__doc__<EOL>return wrapper<EOL>

Instance method decorator to convert an optional file keyword
argument into an actual value, whether it be a passed value, a
value obtained from an io_manager, or sys.stderr.


def get_stdout(self, os_path=None, skip_sub_command=False):

sub_command = None if skip_sub_command else self.stdout_sub_command<EOL>out, path = self._get_out_and_path(<EOL>self.stdout, self.stdout_root, sub_command, os_path)<EOL>if hasattr(out, '<STR_LIT>'):<EOL><INDENT>return out.stdin<EOL><DEDENT>return out<EOL>

Returns a stdout-suitable file-like object based on the
optional os_path and optionally skipping any configured
sub-command.


def get_stdin(self, os_path=None, skip_sub_command=False):

sub_command = None if skip_sub_command else self.stdin_sub_command<EOL>inn, path = self._get_in_and_path(<EOL>self.stdin, self.stdin_root, sub_command, os_path)<EOL>if hasattr(inn, '<STR_LIT>'):<EOL><INDENT>return inn.stdout<EOL><DEDENT>return inn<EOL>

Returns a stdin-suitable file-like object based on the
optional os_path and optionally skipping any configured
sub-command.


if self._pool:<EOL><INDENT>self._pool.waitall()<EOL><DEDENT>

Blocks until all currently pending functions have finished.


def get_location_observation(lat, lng, token):

req = requests.get(<EOL>API_ENDPOINT_GEO % (lat, lng),<EOL>params={<EOL>'<STR_LIT>': token<EOL>})<EOL>if req.status_code == <NUM_LIT:200> and req.json()["<STR_LIT:status>"] == "<STR_LIT>":<EOL><INDENT>return parse_observation_response(req.json()["<STR_LIT:data>"])<EOL><DEDENT>return {}<EOL>


def create_joining_subplan(<EOL>pipeline_def, solid, join_step_key, parallel_steps, parallel_step_output<EOL>):

check.inst_param(pipeline_def, '<STR_LIT>', PipelineDefinition)<EOL>check.inst_param(solid, '<STR_LIT>', Solid)<EOL>check.str_param(join_step_key, '<STR_LIT>')<EOL>check.list_param(parallel_steps, '<STR_LIT>', of_type=ExecutionStep)<EOL>check.str_param(parallel_step_output, '<STR_LIT>')<EOL>for parallel_step in parallel_steps:<EOL><INDENT>check.invariant(parallel_step.has_step_output(parallel_step_output))<EOL><DEDENT>join_step = create_join_step(<EOL>pipeline_def, solid, join_step_key, parallel_steps, parallel_step_output<EOL>)<EOL>output_name = join_step.step_outputs[<NUM_LIT:0>].name<EOL>return ExecutionValueSubplan(<EOL>parallel_steps + [join_step], StepOutputHandle.from_step(join_step, output_name)<EOL>)<EOL>

This captures a common pattern of fanning out a single value to N steps,
where each step has similar structure. The strict requirement here is that each step
must provide an output named the parameters parallel_step_output.

This takes those steps and then uses a join node to coalesce them so that downstream
steps can depend on a single output.

Currently the join step just does a passthrough with no computation. It remains
to be seen if there should be any work or verification done in this step, especially
in multi-process environments that require persistence between steps.


check.type_param(existing_type, '<STR_LIT>')<EOL>check.opt_str_param(name, '<STR_LIT:name>')<EOL>check.opt_str_param(description, '<STR_LIT:description>')<EOL>check.opt_inst_param(input_schema, '<STR_LIT>', InputSchema)<EOL>check.opt_inst_param(output_schema, '<STR_LIT>', OutputSchema)<EOL>check.opt_inst_param(serialization_strategy, '<STR_LIT>', SerializationStrategy)<EOL>storage_plugins = check.opt_dict_param(storage_plugins, '<STR_LIT>')<EOL>if serialization_strategy is None:<EOL><INDENT>serialization_strategy = PickleSerializationStrategy()<EOL><DEDENT>name = existing_type.__name__ if name is None else name<EOL>return _decorate_as_dagster_type(<EOL>existing_type,<EOL>key=name,<EOL>name=name,<EOL>description=description,<EOL>input_schema=input_schema,<EOL>output_schema=output_schema,<EOL>serialization_strategy=serialization_strategy,<EOL>storage_plugins=storage_plugins,<EOL>)<EOL>

Takes a python cls and creates a type for it in the Dagster domain.

Args:
    existing_type (cls)
        The python type you want to project in to the Dagster type system.
    name (Optional[str]):
    description (Optiona[str]):
    input_schema (Optional[InputSchema]):
        An instance of a class that inherits from :py:class:`InputSchema` that
        can map config data to a value of this type.

    output_schema (Optiona[OutputSchema]):
        An instance of a class that inherits from :py:class:`OutputSchema` that
        can map config data to persisting values of this type.

    serialization_strategy (Optional[SerializationStrategy]):
        The default behavior for how to serialize this value for
        persisting between execution steps.

    storage_plugins (Optional[Dict[RunStorageMode, TypeStoragePlugin]]):
        Storage type specific overrides for the serialization strategy.
        This allows for storage specific optimzations such as effecient
        distributed storage on S3.


config_type = resolve_config_cls_arg(config_cls)<EOL>return lambda func: _create_output_schema(config_type, func)<EOL>

A decorator for a annotating a function that can take a ``config_value``
and an instance of a custom type and materialize it.

Args:
    config_cls (Any):


def materialize_runtime_value(self, _context, _config_value, _runtime_value):

How to materialize a runtime value given configuration.


config_type = resolve_config_cls_arg(config_cls)<EOL>return lambda func: _create_input_schema(config_type, func)<EOL>

A decorator for annotating a function that can turn a ``config_value`` in to
an instance of a custom type.

Args:
    config_cls (Any):


return WrappingListType(inner_type)<EOL>

Validates at runtime that the value is ``List[inner_type]``.

Args:
    inner_type (DagsterType)


def NamedSelector(name, fields, description=None, type_attributes=DEFAULT_TYPE_ATTRIBUTES):

A :py:class`Selector` with a name, allowing it to be referenced by that name.

Args:
    name (str):
    fields (Dict[str, Field])


check_user_facing_fields_dict(fields, '<STR_LIT>')<EOL>class _Dict(_ConfigComposite):<EOL><INDENT>def __init__(self):<EOL><INDENT>key = '<STR_LIT>' + str(DictCounter.get_next_count())<EOL>super(_Dict, self).__init__(<EOL>name=None,<EOL>key=key,<EOL>fields=fields,<EOL>description='<STR_LIT>',<EOL>type_attributes=ConfigTypeAttributes(is_builtin=True),<EOL>)<EOL><DEDENT><DEDENT>return _Dict<EOL>

Schema for configuration data with string keys and typed values via :py:class:`Field` .

Args:
    fields (Dict[str, Field])


Record a stream of event records to json


check.inst_param(context_params, '<STR_LIT>', ExecutionContext)<EOL>context_definition = PipelineContextDefinition(context_fn=lambda *_args: context_params)<EOL>return {DEFAULT_CONTEXT_NAME: context_definition}<EOL>

Create a context definition from a pre-existing context. This can be useful
        in testing contexts where you may want to create a context manually and then
        pass it into a one-off PipelineDefinition

        Args:
            context (ExecutionContext): The context that will provided to the pipeline.
        Returns:
            PipelineContextDefinition: The passthrough context definition.


return self.name if self.name else '<STR_LIT>'<EOL>

Name suitable for exception messages, logging etc. If pipeline
        is unnamed the method with return "<<unnamed>>".

        Returns:
            str: Display name of pipeline


pipelines = list(map(self.get_pipeline, self.pipeline_dict.keys()))<EOL>self._construct_solid_defs(pipelines)<EOL>return pipelines<EOL>

Return all pipelines as a list

        Returns:
            List[PipelineDefinition]:


def lambda_solid(name=None, inputs=None, output=None, description=None):

output = output or OutputDefinition()<EOL>if callable(name):<EOL><INDENT>check.invariant(inputs is None)<EOL>check.invariant(description is None)<EOL>return _LambdaSolid(output=output)(name)<EOL><DEDENT>return _LambdaSolid(name=name, inputs=inputs, output=output, description=description)<EOL>

(decorator) Create a simple solid.

    This shortcut allows the creation of simple solids that do not require
    configuration and whose implementations do not require a context.

    Lambda solids take inputs and produce a single output. The body of the function
    should return a single value.

    Args:
        name (str): Name of solid.
        inputs (list[InputDefinition]): List of inputs.
        output (OutputDefinition): The output of the solid. Defaults to ``OutputDefinition()``.
        description (str): Solid description.

    Examples:

        .. code-block:: python

            @lambda_solid
            def hello_world():
                return 'hello'

            @lambda_solid(inputs=[InputDefinition(name='foo')])
            def hello_world(foo):
                return foo


def SystemNamedDict(name, fields, description=None):

return NamedDict(name, fields, description, ConfigTypeAttributes(is_system_config=True))<EOL>

A SystemNamedDict object is simply a NamedDict intended for internal (dagster) use.


@contextmanager<EOL>def user_code_context_manager(user_fn, error_cls, msg):

check.callable_param(user_fn, '<STR_LIT>')<EOL>check.subclass_param(error_cls, '<STR_LIT>', DagsterUserCodeExecutionError)<EOL>with user_code_error_boundary(error_cls, msg):<EOL><INDENT>thing_or_gen = user_fn()<EOL>gen = _ensure_gen(thing_or_gen)<EOL>try:<EOL><INDENT>thing = next(gen)<EOL><DEDENT>except StopIteration:<EOL><INDENT>check.failed('<STR_LIT>')<EOL><DEDENT>yield thing<EOL>stopped = False<EOL>try:<EOL><INDENT>next(gen)<EOL><DEDENT>except StopIteration:<EOL><INDENT>stopped = True<EOL><DEDENT>check.invariant(stopped, '<STR_LIT>')<EOL><DEDENT>

Wraps the output of a user provided function that may yield or return a value and
    returns a generator that asserts it only yields a single value.


def execute_pipeline_iterator(pipeline, environment_dict=None, run_config=None):

check.inst_param(pipeline, '<STR_LIT>', PipelineDefinition)<EOL>environment_dict = check.opt_dict_param(environment_dict, '<STR_LIT>')<EOL>run_config = check_run_config_param(run_config)<EOL>environment_config = create_environment_config(pipeline, environment_dict)<EOL>intermediates_manager = construct_intermediates_manager(<EOL>run_config, environment_config, pipeline<EOL>)<EOL>with _pipeline_execution_context_manager(<EOL>pipeline, environment_config, run_config, intermediates_manager<EOL>) as pipeline_context:<EOL><INDENT>return _execute_pipeline_iterator(pipeline_context)<EOL><DEDENT>

Returns iterator that yields :py:class:`SolidExecutionResult` for each
    solid executed in the pipeline.

    This is intended to allow the caller to do things between each executed
    node. For the 'synchronous' API, see :py:func:`execute_pipeline`.

    Parameters:
      pipeline (PipelineDefinition): Pipeline to run
      environment_dict (dict): The enviroment configuration that parameterizes this run
      run_config (RunConfig): Configuration for how this pipeline will be executed

    Returns:
      Iterator[DagsterEvent]


any_success = False<EOL>for step_event in itertools.chain(<EOL>self.input_expectations, self.output_expectations, self.transforms<EOL>):<EOL><INDENT>if step_event.event_type == DagsterEventType.STEP_FAILURE:<EOL><INDENT>return False<EOL><DEDENT>if step_event.event_type == DagsterEventType.STEP_SUCCESS:<EOL><INDENT>any_success = True<EOL><DEDENT><DEDENT>return any_success<EOL>

Whether the solid execution was successful


return all(<EOL>[<EOL>step_event.event_type == DagsterEventType.STEP_SKIPPED<EOL>for step_event in itertools.chain(<EOL>self.input_expectations, self.output_expectations, self.transforms<EOL>)<EOL>]<EOL>)<EOL>


def block(self, text, prefix='<STR_LIT>'):

wrapper = TextWrapper(<EOL>width=self.line_length - len(self.current_indent_str),<EOL>initial_indent=prefix,<EOL>subsequent_indent=prefix,<EOL>break_long_words=False,<EOL>break_on_hyphens=False,<EOL>)<EOL>for line in wrapper.wrap(text):<EOL><INDENT>self.line(line)<EOL><DEDENT>


def dict_param(obj, param_name, key_type=None, value_type=None):

if not isinstance(obj, dict):<EOL><INDENT>raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))<EOL><DEDENT>if not (key_type or value_type):<EOL><INDENT>return obj<EOL><DEDENT>return _check_key_value_types(obj, key_type, value_type)<EOL>

Ensures argument obj is a native Python dictionary, raises an exception if not, and otherwise
    returns obj.


def opt_dict_param(obj, param_name, key_type=None, value_type=None, value_class=None):

if obj is not None and not isinstance(obj, dict):<EOL><INDENT>raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))<EOL><DEDENT>if not obj:<EOL><INDENT>return {}<EOL><DEDENT>if value_class:<EOL><INDENT>return _check_key_value_types(obj, key_type, value_type=value_class, value_check=issubclass)<EOL><DEDENT>return _check_key_value_types(obj, key_type, value_type)<EOL>

Ensures argument obj is either a dictionary or None; if the latter, instantiates an empty
    dictionary.


master_url = Field(<EOL>String,<EOL>description='<STR_LIT>',<EOL>is_optional=False,<EOL>)<EOL>deploy_mode = Field(<EOL>SparkDeployMode,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>application_jar = Field(<EOL>Path,<EOL>description='''<STR_LIT>''',<EOL>is_optional=False,<EOL>)<EOL>application_arguments = Field(<EOL>String,<EOL>description='<STR_LIT>',<EOL>is_optional=True,<EOL>)<EOL>spark_home = Field(<EOL>String,<EOL>description='<STR_LIT>',<EOL>is_optional=True,<EOL>)<EOL>spark_outputs = Field(List(String), description='<STR_LIT>')<EOL>return Field(<EOL>Dict(<EOL>fields={<EOL>'<STR_LIT>': master_url,<EOL>'<STR_LIT>': deploy_mode,<EOL>'<STR_LIT>': application_jar,<EOL>'<STR_LIT>': spark_config(),<EOL>'<STR_LIT>': spark_home,<EOL>'<STR_LIT>': application_arguments,<EOL>'<STR_LIT>': spark_outputs,<EOL>}<EOL>)<EOL>)<EOL>

Spark configuration.

    See the Spark documentation for reference:
        https://spark.apache.org/docs/latest/submitting-applications.html


sf = _define_shared_fields()<EOL>allow_large_results = Field(<EOL>Bool,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>default_dataset = Field(<EOL>Dataset,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>destination = Field(<EOL>Table,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>dry_run = Field(<EOL>Bool,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>flatten_results = Field(<EOL>Bool,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>maximum_billing_tier = Field(<EOL>Int,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>maximum_bytes_billed = Field(<EOL>Int,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>priority = Field(<EOL>BQPriority,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>query_parameters = Field(<EOL>List(String),<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>use_legacy_sql = Field(<EOL>Bool,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>use_query_cache = Field(<EOL>Bool,<EOL>description='''<STR_LIT>''',<EOL>is_optional=True,<EOL>)<EOL>return Field(<EOL>Dict(<EOL>fields={<EOL>'<STR_LIT>': Field(<EOL>Dict(<EOL>fields={<EOL>'<STR_LIT>': allow_large_results,<EOL>'<STR_LIT>': sf['<STR_LIT>'],<EOL>'<STR_LIT>': sf['<STR_LIT>'],<EOL>'<STR_LIT>': default_dataset,<EOL>'<STR_LIT>': destination,<EOL>'<STR_LIT>': sf[<EOL>'<STR_LIT>'<EOL>],<EOL>'<STR_LIT>': dry_run,<EOL>'<STR_LIT>': flatten_results,<EOL>'<STR_LIT>': maximum_billing_tier,<EOL>'<STR_LIT>': maximum_bytes_billed,<EOL>'<STR_LIT>': priority,<EOL>'<STR_LIT>': query_parameters,<EOL>'<STR_LIT>': sf['<STR_LIT>'],<EOL>'<STR_LIT>': sf['<STR_LIT>'],<EOL>'<STR_LIT>': use_legacy_sql,<EOL>'<STR_LIT>': use_query_cache,<EOL>'<STR_LIT>': sf['<STR_LIT>'],<EOL>}<EOL>)<EOL>)<EOL>}<EOL>),<EOL>description='<STR_LIT>',<EOL>)<EOL>

See:
    https://googleapis.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.QueryJobConfig.html


return re.match(<EOL>r'<STR_LIT>'<EOL>+ RE_PROJECT  <EOL>+ r'<STR_LIT>'  <EOL>+ RE_DS_TABLE  <EOL>+ r'<STR_LIT>'  <EOL>+ RE_DS_TABLE  <EOL>+ r'<STR_LIT>'  <EOL>+ RE_DS_TABLE  <EOL>+ r'<STR_LIT>'  <EOL>+ RE_DS_TABLE  <EOL>+ r'<STR_LIT:$>',<EOL>config_value,<EOL>)<EOL>

Tables must be of form "project.dataset.table" or "dataset.table"


def _format_config_subdict(config, current_indent=<NUM_LIT:0>):<EOL><INDENT>check.dict_param(config, '<STR_LIT>', key_type=str)<EOL>printer = IndentingStringIoPrinter(indent_level=<NUM_LIT:2>, current_indent=current_indent)<EOL>printer.line('<STR_LIT:{>')<EOL>n_elements = len(config)<EOL>for i, key in enumerate(sorted(config, key=lambda x: x[<NUM_LIT:0>])):<EOL><INDENT>value = config[key]<EOL>with printer.with_indent():<EOL><INDENT>formatted_value = (<EOL>_format_config_item(value, current_indent=printer.current_indent)<EOL>.lstrip('<STR_LIT:U+0020>')<EOL>.rstrip('<STR_LIT:\n>')<EOL>)<EOL>printer.line(<EOL>'<STR_LIT>'.format(<EOL>key=key,<EOL>formatted_value=formatted_value,<EOL>comma='<STR_LIT:U+002C>' if i != n_elements - <NUM_LIT:1> else '<STR_LIT>',<EOL>)<EOL>)<EOL><DEDENT><DEDENT>printer.line('<STR_LIT:}>')<EOL>return printer.read()<EOL><DEDENT>def _format_config_sublist(config, current_indent=<NUM_LIT:0>):<EOL><INDENT>printer = IndentingStringIoPrinter(indent_level=<NUM_LIT:2>, current_indent=current_indent)<EOL>printer.line('<STR_LIT:[>')<EOL>n_elements = len(config)<EOL>for i, value in enumerate(config):<EOL><INDENT>with printer.with_indent():<EOL><INDENT>formatted_value = (<EOL>_format_config_item(value, current_indent=printer.current_indent)<EOL>.lstrip('<STR_LIT:U+0020>')<EOL>.rstrip('<STR_LIT:\n>')<EOL>)<EOL>printer.line(<EOL>'<STR_LIT>'.format(<EOL>formatted_value=formatted_value, comma='<STR_LIT:U+002C>' if i != n_elements - <NUM_LIT:1> else '<STR_LIT>'<EOL>)<EOL>)<EOL><DEDENT><DEDENT>printer.line('<STR_LIT:]>')<EOL>return printer.read()<EOL><DEDENT>def _format_config_item(config, current_indent=<NUM_LIT:0>):<EOL><INDENT>printer = IndentingStringIoPrinter(indent_level=<NUM_LIT:2>, current_indent=current_indent)<EOL>if isinstance(config, dict):<EOL><INDENT>return _format_config_subdict(config, printer.current_indent)<EOL><DEDENT>elif isinstance(config, list):<EOL><INDENT>return _format_config_sublist(config, printer.current_indent)<EOL><DEDENT>elif isinstance(config, bool):<EOL><INDENT>return repr(config).lower()<EOL><DEDENT>else:<EOL><INDENT>return repr(config).replace('<STR_LIT>', '<STR_LIT:">')<EOL><DEDENT><DEDENT>check.dict_param(config, '<STR_LIT>', key_type=str)<EOL>if not isinstance(config, dict):<EOL><INDENT>check.failed('<STR_LIT>'.format(item=repr(config)))<EOL><DEDENT>return _format_config_subdict(config)<EOL>

This recursive descent thing formats a config dict for GraphQL.


check_new_version(version)<EOL>set_new_version(version)<EOL>commit_new_version(version)<EOL>set_git_tag(version)<EOL>

Tags all submodules for a new release.

    Ensures that git tags, as well as the version.py files in each submodule, agree and that the
    new version is strictly greater than the current version. Will fail if the new version
    is not an increment (following PEP 440). Creates a new git tag and commit.


<EOL>return spawn.find_executable(exe)<EOL>

Uses distutils to look for an executable, mimicking unix which


config = configparser.ConfigParser()<EOL>config.read(path)<EOL>return config<EOL>

Make a config parser for the given config file.

    Return a :class:`configparser.ConfigParser` instance with the given file
    loaded.

    :param path:
        Path to the config file to read.


servers = self._read_index_servers()<EOL>repo_config = self._find_repo_config(servers, repository)<EOL>return repo_config<EOL>

Get config dictionary for the given repository.

        If the repository section is not found in the config file,
        return ``None``.  If the file is invalid, raise
        :exc:`configparser.Error`.

        Otherwise return a dictionary with:

        * ``'repository'`` -- the repository URL
        * ``'username'`` -- username for authentication
        * ``'password'`` -- password for authentication

        :param repository:
            Name or URL of the repository to find in the ``.pypirc`` file.
            The repository section must be defined in the config file.


try:<EOL><INDENT>os.makedirs(newdir, mode)<EOL><DEDENT>except OSError as err:<EOL><INDENT>if err.errno != errno.EEXIST or not os.path.isdir(newdir):<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

The missing mkdir -p functionality in os.


bucket = context.solid_config['<STR_LIT>']<EOL>key = context.solid_config['<STR_LIT:key>']<EOL>context.resources.s3.put_object(<EOL>Bucket=bucket, Body=file_obj.read(), Key=key, **(context.solid_config.get('<STR_LIT>') or {})<EOL>)<EOL>yield Result(bucket, '<STR_LIT>')<EOL>yield Result(key, '<STR_LIT:key>')<EOL>

Upload a file to s3.

    Args:
        info (ExpectationExecutionInfo): Must expose a boto3 S3 client as its `s3` resource.

    Returns:
        (str, str):
            The bucket and key to which the file was uploaded.


def nonce_solid(name, n_inputs, n_outputs):

@solid(<EOL>name=name,<EOL>inputs=[<EOL>InputDefinition(name='<STR_LIT>'.format(i)) for i in range(n_inputs)<EOL>],<EOL>outputs=[<EOL>OutputDefinition(name='<STR_LIT>'.format(i))<EOL>for i in range(n_outputs)<EOL>],<EOL>)<EOL>def solid_fn(context, **_kwargs):<EOL><INDENT>for i in range(<NUM_LIT:200>):<EOL><INDENT>time.sleep(<NUM_LIT>)<EOL>if i % <NUM_LIT:1000> == <NUM_LIT>:<EOL><INDENT>context.log.error(<EOL>'<STR_LIT>'.format(<EOL>i=i, name=name<EOL>)<EOL>)<EOL><DEDENT>elif i % <NUM_LIT:100> == <NUM_LIT:0>:<EOL><INDENT>context.log.warning(<EOL>'<STR_LIT>'.format(<EOL>i=i, name=name<EOL>)<EOL>)<EOL><DEDENT>elif i % <NUM_LIT:10> == <NUM_LIT:0>:<EOL><INDENT>context.log.info(<EOL>'<STR_LIT>'.format(<EOL>i=i, name=name<EOL>)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>context.log.debug(<EOL>'<STR_LIT>'.format(<EOL>i=i, name=name<EOL>)<EOL>)<EOL><DEDENT><DEDENT>return MultipleResults.from_dict(<EOL>{'<STR_LIT>'.format(i): '<STR_LIT:foo>' for i in range(n_outputs)}<EOL>)<EOL><DEDENT>return solid_fn<EOL>

Creates a solid with the given number of (meaningless) inputs and outputs.

    Config controls the behavior of the nonce solid.


num_urls = len(urls)<EOL>for url in urls:<EOL><INDENT>yield (url, rank / num_urls)<EOL><DEDENT>

Calculates URL contributions to the rank of other URLs.


parts = re.split(r'<STR_LIT>', urls)<EOL>return parts[<NUM_LIT:0>], parts[<NUM_LIT:1>]<EOL>

Parses a urls pair string into urls pair.


return list(<EOL>filter(<EOL>lambda x: x.get('<STR_LIT:type>') != '<STR_LIT>',  <EOL>self._post(<EOL>request=ApiActions.LIST.value,<EOL>uri=ApiUri.ACTIONS.value,<EOL>).get('<STR_LIT>')<EOL>)<EOL>)<EOL>

Get all current alerts

:return: All alerts
:rtype: list of dict

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


data = {<EOL>'<STR_LIT>': occurrence_frequency_count or <NUM_LIT:1>,<EOL>'<STR_LIT>': occurrence_frequency_unit or '<STR_LIT>',<EOL>'<STR_LIT>': alert_frequency_count or <NUM_LIT:1>,<EOL>'<STR_LIT>': alert_frequency_unit or '<STR_LIT>',<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': True,<EOL>}<EOL>data.update(alert_config.args())<EOL>return self._post(<EOL>request=ApiActions.CREATE.value,<EOL>uri=ApiUri.ACTIONS.value,<EOL>params=data<EOL>)<EOL>

Create a new alert

:param alert_config: A list of AlertConfig classes (Ex:
    ``[EmailAlertConfig('me@mydomain.com')]``)
:type alert_config: list of
    :class:`PagerDutyAlertConfig<logentries_api.alerts.PagerDutyAlertConfig>`,
    :class:`WebHookAlertConfig<logentries_api.alerts.WebHookAlertConfig>`,
    :class:`EmailAlertConfig<logentries_api.alerts.EmailAlertConfig>`,
    :class:`SlackAlertConfig<logentries_api.alerts.SlackAlertConfig>`, or
    :class:`HipChatAlertConfig<logentries_api.alerts.HipChatAlertConfig>`

:param occurrence_frequency_count: How many times per
    ``alert_frequency_unit`` for a match before issuing an alert.
    Defaults to 1
:type occurrence_frequency_count: int

:param occurrence_frequency_unit: The time period to monitor for sending
    an alert. Must be 'day', or 'hour'. Defaults to 'hour'
:type occurrence_frequency_unit: str

:param alert_frequency_count: How many times per
    ``alert_frequency_unit`` to issue an alert. Defaults to 1
:type alert_frequency_count: int

:param alert_frequency_unit: How often to regulate sending alerts.
    Must be 'day', or 'hour'. Defaults to 'hour'
:type alert_frequency_unit: str

:returns: The response of your post
:rtype: dict

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


data = {<EOL>'<STR_LIT:id>': alert['<STR_LIT:id>'],<EOL>'<STR_LIT:args>': alert['<STR_LIT:args>'],<EOL>'<STR_LIT>': alert['<STR_LIT>'],<EOL>'<STR_LIT>': alert['<STR_LIT>'],<EOL>'<STR_LIT>': alert['<STR_LIT>'],<EOL>'<STR_LIT>': alert['<STR_LIT>'],<EOL>'<STR_LIT>': alert['<STR_LIT>'],<EOL>'<STR_LIT>': alert['<STR_LIT>'],<EOL>'<STR_LIT:type>': alert['<STR_LIT:type>'],<EOL>}<EOL>return self._post(<EOL>request=ApiActions.UPDATE.value,<EOL>uri=ApiUri.ACTIONS.value,<EOL>params=data<EOL>)<EOL>

Update an alert

:param alert: The data to update. Must include keys:

    * id (str)
    * rate_count (int)
    * rate_range (str): 'day' or 'hour'
    * limit_count (int)
    * limit_range (str): 'day' or 'hour'
    * type (str)
    * schedule (list)
    * args (dict)
:type alert: dict

Example:

.. code-block:: python

    Alert().update(
        alert={
            'id': 'd9d4596e-49e4-4135-b3b3-847f9e7c1f43',
            'args': {'direct': 'you@example.com'},
            'rate_count': 1,
            'rate_range': 'hour',
            'limit_count': 1,
            'limit_range': 'hour',
            'schedule': [],
            'enabled': True,
            'type': 'mailto',
        }
    )

:return:
:rtype: dict


return self._post(<EOL>request=ApiActions.DELETE.value,<EOL>uri=ApiUri.ACTIONS.value,<EOL>params={'<STR_LIT:id>': id}<EOL>)<EOL>

Delete the specified alert

:param id: the alert's ID
:type id: str

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


return self._post(<EOL>request='<STR_LIT:list>',<EOL>uri=ApiUri.TAGS.value,<EOL>).get('<STR_LIT>')<EOL>

Get all current labels

:return: The Logentries API response
:rtype: list of dict

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


tag_url = '<STR_LIT>'<EOL>self._api_delete(<EOL>url=tag_url.format(<EOL>account_id=self.account_id,<EOL>tag_id=tag_id<EOL>)<EOL>)<EOL>

Delete the specified InactivityAlert

:param tag_id: The tag ID to delete
:type tag_id: str

:raises: This will raise a
    :class:`ServerException <logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


change = '<STR_LIT>'.format(<EOL>pos='<STR_LIT:+>' if increase_positive else '<STR_LIT:->',<EOL>change=str(percentage_change)<EOL>)<EOL>query_response = self._create_scheduled_query(<EOL>query=query,<EOL>change=change,<EOL>scope_unit=scope_unit,<EOL>scope_count=scope_count,<EOL>)<EOL>scheduled_query_id = query_response.get('<STR_LIT>', {}).get('<STR_LIT:id>')<EOL>tag_data = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': [<EOL>alert_report.to_dict()<EOL>for alert_report<EOL>in alert_reports<EOL>],<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': scheduled_query_id,<EOL>'<STR_LIT>': [<EOL>{'<STR_LIT:id>': log}<EOL>for log<EOL>in logs<EOL>],<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:type>': '<STR_LIT>'<EOL>}<EOL>}<EOL>tag_data['<STR_LIT>'].update(trigger_config.to_dict())<EOL>tag_url = '<STR_LIT>'.format(<EOL>account_id=self.account_id<EOL>)<EOL>return self._api_post(<EOL>url=tag_url,<EOL>data=json.dumps(tag_data, sort_keys=True),<EOL>)<EOL>

Create an anomaly alert. This call makes 2 requests, one to create a
"scheduled_query", and another to create the alert.

:param name: The name for the alert
:type name: str

:param query: The `LEQL`_ query to use for detecting anomalies. Must
    result in a numerical value, so it should look something like
    ``where(...) calculate(COUNT)``
:type query: str

:param scope_count: How many ``scope_unit`` s to inspect for detecting
    an anomaly
:type scope_count: int

:param scope_unit: How far to look back in detecting an anomaly. Must
    be one of "hour", "day", or "week"
:type scope_unit: str

:param increase_positive: Detect a positive increase for the anomaly. A
    value of ``False`` results in detecting a decrease for the anomaly.
:type increase_positive: bool

:param percentage_change: The percentage of change to detect. Must be a
    number between 0 and 100 (inclusive).
:type percentage_change: int

:param trigger_config: A AlertTriggerConfig describing how far back to
    look back to compare to the anomaly scope.
:type trigger_config: :class:`AlertTriggerConfig<logentries_api.special_alerts.AlertTriggerConfig>`

:param logs: A list of log UUID's. (The 'key' key of a log)
:type logs: list of str

:param alert_reports: A list of AlertReportConfig to send alerts to
:type alert_reports: list of
    :class:`AlertReportConfig<logentries_api.special_alerts.AlertReportConfig>`

:return: The API response of the alert creation
:rtype: dict

:raises: This will raise a
    :class:`ServerException <logentries_api.exceptions.ServerException>`
    if there is an error from Logentries

.. _Leql: https://blog.logentries.com/2015/06/introducing-leql/


def create(self, name, patterns, logs, trigger_config, alert_reports):

data = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': [<EOL>alert_report.to_dict()<EOL>for alert_report<EOL>in alert_reports<EOL>],<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': patterns,<EOL>'<STR_LIT>': [<EOL>{'<STR_LIT:id>': log}<EOL>for log<EOL>in logs<EOL>],<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:type>': '<STR_LIT>'<EOL>}<EOL>}<EOL>data['<STR_LIT>'].update(trigger_config.to_dict())<EOL>return self._api_post(<EOL>url=self.url_template.format(account_id=self.account_id),<EOL>data=json.dumps(data, sort_keys=True)<EOL>)<EOL>

Create an inactivity alert

:param name: A name for the inactivity alert
:type name: str

:param patterns: A list of regexes to match
:type patterns: list of str

:param logs: A list of log UUID's. (The 'key' key of a log)
:type logs: list of str

:param trigger_config: A AlertTriggerConfig describing how far back to
    look for inactivity.
:type trigger_config: :class:`AlertTriggerConfig<logentries_api.special_alerts.AlertTriggerConfig>`

:param alert_reports: A list of AlertReportConfigs to send alerts to
:type alert_reports: list of
    :class:`AlertReportConfig<logentries_api.special_alerts.AlertReportConfig>`

:return: The API response
:rtype: dict

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


def _create_scheduled_query(self, query, change, scope_unit, scope_count):

query_data = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': query,<EOL>'<STR_LIT>': '<STR_LIT:%>',<EOL>'<STR_LIT>': change,<EOL>'<STR_LIT>': scope_unit.title(),<EOL>'<STR_LIT>': scope_count,<EOL>}<EOL>}<EOL>query_url = '<STR_LIT>'<EOL>return self._api_post(<EOL>url=query_url.format(account_id=self.account_id),<EOL>data=json.dumps(query_data, sort_keys=True)<EOL>)<EOL>


def _post(self, request, uri, params=None):

request_data = {<EOL>'<STR_LIT>': self.account_key,<EOL>'<STR_LIT>': self.account_key,<EOL>'<STR_LIT>': request,<EOL>}<EOL>request_data.update(params or {})<EOL>response = requests.post(<EOL>url='<STR_LIT>'.format(uri),<EOL>headers=self.headers,<EOL>data=json.dumps(request_data)<EOL>)<EOL>if not response.ok:<EOL><INDENT>raise ServerException(<EOL>'<STR_LIT>'.format(response.status_code, response.text))<EOL><DEDENT>return response.json()<EOL>

A wrapper for posting things.

:param request: The request type. Must be one of the
    :class:`ApiActions<logentries_api.base.ApiActions>`
:type request: str

:param uri: The API endpoint to hit. Must be one of
    :class:`ApiUri<logentries_api.base.ApiUri>`
:type uri: str

:param params: A dictionary of supplemental kw args
:type params: dict

:returns: The response of your post
:rtype: dict

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


return {<EOL>'<STR_LIT:args>': {<EOL>'<STR_LIT>': self.address,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>},<EOL>'<STR_LIT:type>': AlertTypes.EMAIL.value<EOL>}<EOL>


super(WebHookAlertConfig, self).__init__(url=url)<EOL>


raise NotImplementedError<EOL>

Must return a dictionary with an `args` for the alert, and a `type` key
with the type
:rtype: dict


super(EmailAlertConfig, self).__init__(address=address)<EOL>


return {<EOL>'<STR_LIT:args>': {<EOL>'<STR_LIT:url>': self.url,<EOL>},<EOL>'<STR_LIT:type>': AlertTypes.SLACK.value<EOL>}<EOL>


response = requests.get(self.base_url)<EOL>if not response.ok:<EOL><INDENT>raise ServerException(<EOL>'<STR_LIT>'.format(response.status_code, response.text))<EOL><DEDENT>return {<EOL>host.get('<STR_LIT:name>'): [<EOL>log.get('<STR_LIT:key>')<EOL>for log<EOL>in host.get('<STR_LIT>')]<EOL>for host<EOL>in response.json().get('<STR_LIT:list>')<EOL>}<EOL>

Get all log sets

:return: Returns a dictionary where the key is the hostname or log set,
    and the value is a list of the log keys
:rtype: dict

:raises: This will raise a
    :class:`ServerException<logentries_api.exceptions.ServerException>`
    if there is an error from Logentries


_implementations[name] = impl<EOL>

Register your own implementation,
    it also override registered implementation without any check.


formats = {'<STR_LIT>': json,<EOL>'<STR_LIT>': pickle,<EOL>'<STR_LIT>': Storable}<EOL>return formats[codec]<EOL>

Create a serializer that support loads/dumps methods.
json and pickle are fully supported.
storable support read only.


def save_session(self, request=None, response=None):

if not self._session_key:<EOL><INDENT>return<EOL><DEDENT>if self._session_data is None:  <EOL><INDENT>self.client.delete(self._session_key)<EOL>return<EOL><DEDENT>self.client.set(self._session_key, self._session_data)<EOL>

Save the session in the key value store, in case a session
        has been found


def update_session_token(self, header_name, value):

if self._session_key:<EOL><INDENT>self.client.delete(self._session_key)<EOL><DEDENT>self._session_key = '<STR_LIT>' % (header_name, value)<EOL>

Create a session from the givent header name


def add_timestamp(logger_class, log_method, event_dict):

event_dict['<STR_LIT>'] = calendar.timegm(time.gmtime())<EOL>return event_dict<EOL>


def add_unique_id(logger_class, log_method, event):

event['<STR_LIT:id>'] = dna.utils.generate_unique_id()<EOL>return event<EOL>


@functools.wraps(resource)<EOL>def decorated(*args, **kwargs):<EOL><INDENT>'''<STR_LIT>'''<EOL>auth = flask.request.authorization<EOL>user = check_credentials(auth.username, auth.password)<EOL>if not auth or user is None:<EOL><INDENT>log.warn('<STR_LIT>', credentials=auth)<EOL>return auth_failed()<EOL><DEDENT>log.info('<STR_LIT>', credentials=auth)<EOL>flask.g.user = user<EOL>return resource(*args, **kwargs)<EOL><DEDENT>return decorated<EOL>

Flask decorator protecting ressources using username/password scheme


return flask.Response(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', <NUM_LIT>,<EOL>{'<STR_LIT>': '<STR_LIT>'})<EOL>


user = models.User.objects(api_key=token).first()<EOL>return user or None<EOL>

Verify http header token authentification


return '<STR_LIT>'.format(dna.utils.Version(full_version).major, resource)<EOL>

>>> # Harmonize api endpoints
>>> # __version__ should be like major.minor.fix
>>> from my_app import __version__
>>> api_url(__version__, '/some/endpoint')
/v0/some/endpoint


code = <NUM_LIT:200><EOL>if worker_id == '<STR_LIT:all>':<EOL><INDENT>report = {'<STR_LIT>': [{<EOL>'<STR_LIT:id>': job,<EOL>'<STR_LIT>': self._inspect_worker(job)}<EOL>for job in self.jobs]<EOL>}<EOL><DEDENT>elif worker_id in self.jobs:<EOL><INDENT>report = {<EOL>'<STR_LIT:id>': worker_id,<EOL>'<STR_LIT>': self._inspect_worker(worker_id)<EOL>}<EOL><DEDENT>else:<EOL><INDENT>report = {'<STR_LIT:error>': '<STR_LIT>'.format(worker_id)}<EOL>code = <NUM_LIT><EOL><DEDENT>return flask.jsonify(report), code<EOL>


return functools.partial(self.__call__, obj)<EOL>


return self.func.__doc__<EOL>


try:<EOL><INDENT>module = __import__(mod_path, fromlist=['<STR_LIT>'])<EOL><DEDENT>except ImportError as error:<EOL><INDENT>raise errors.DynamicImportFailed(<EOL>module='<STR_LIT:.>'.join([mod_path, obj_name]), reason=error)<EOL><DEDENT>importlib.reload(module)<EOL>if obj_name is None:<EOL><INDENT>obj = module<EOL><DEDENT>elif hasattr(module, obj_name):<EOL><INDENT>obj = getattr(module, obj_name)<EOL><DEDENT>else:<EOL><INDENT>raise errors.DynamicImportFailed(<EOL>module='<STR_LIT:.>'.join([mod_path, obj_name]),<EOL>reason='<STR_LIT>'.<EOL>format(module.__name__, obj_name))<EOL>return None<EOL><DEDENT>return obj<EOL>

Take a string and return the corresponding module


def generate_random_name(size=<NUM_LIT:8>, chars=string.ascii_lowercase + string.digits):

return '<STR_LIT>'.join(random.choice(chars) for _ in range(size))<EOL>

Create a random name to assign to a node


def iter_osm_notes(feed_limit=<NUM_LIT>, interval=<NUM_LIT>, parse_timestamps=True):

last_seen_guid = None<EOL>while True:<EOL><INDENT>u = urllib2.urlopen('<STR_LIT>' % feed_limit)<EOL>tree = etree.parse(u)<EOL>new_notes = []<EOL>for note_item in tree.xpath('<STR_LIT>'):<EOL><INDENT>title = note_item.xpath('<STR_LIT:title>')[<NUM_LIT:0>].text<EOL>if title.startswith('<STR_LIT>'):<EOL><INDENT>action = '<STR_LIT>'<EOL><DEDENT>elif title.startswith('<STR_LIT>'):<EOL><INDENT>action = '<STR_LIT>'<EOL><DEDENT>elif title.startswith('<STR_LIT>'):<EOL><INDENT>action = '<STR_LIT>'<EOL><DEDENT>guid = note_item.xpath('<STR_LIT>')[<NUM_LIT:0>].text<EOL>if last_seen_guid == guid:<EOL><INDENT>break<EOL><DEDENT>elif last_seen_guid == None:<EOL><INDENT>last_seen_guid = guid<EOL><DEDENT>else:<EOL><INDENT>note_id = int(guid.split('<STR_LIT:/>')[-<NUM_LIT:1>].split('<STR_LIT>')[<NUM_LIT:0>])<EOL>new_notes.append((action, get_note(note_id, parse_timestamps)))<EOL><DEDENT><DEDENT>for note in reversed(new_notes):<EOL><INDENT>yield note<EOL><DEDENT>yield model.Finished(None, None)<EOL>time.sleep(interval)<EOL><DEDENT>

Parses the global OSM Notes feed and yields as much Note information as possible.


@wraps(f)<EOL>def wrapper(*args, **kwargs):<EOL><INDENT>if not wrapper.has_run:<EOL><INDENT>result = f(*args, **kwargs)<EOL>wrapper.has_run = True<EOL>return result<EOL><DEDENT><DEDENT>wrapper.has_run = False<EOL>return wrapper<EOL>

Runs a function (successfully) only once.
    The running can be reset by setting the `has_run` attribute to False


self._validate_key_set()<EOL>self.log.info("<STR_LIT>")<EOL>self.execute_cmd(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'.format(self.project_id)])<EOL>self.execute_cmd(<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:none>', '<STR_LIT>'.format(self.project_id)]<EOL>)<EOL>

Change default authentication to none - which is not existing one.


self._validate_key_set()<EOL>if not GcpAuthenticator.original_account:<EOL><INDENT>GcpAuthenticator.original_account = self.check_output(<EOL>['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'.format(self.project_id)]<EOL>).decode('<STR_LIT:utf-8>')<EOL>self.log.info("<STR_LIT>".format(GcpAuthenticator.original_account))<EOL><DEDENT>

Store authentication as it was originally so it can be restored and revoke
authentication.


return '<STR_LIT>'.format(host)<EOL>

Utility function to generate the get run endpoint given the host.


return '<STR_LIT>'.format(host)<EOL>

Utility function to generate the run now endpoint given the host.


truncated = dropwhile(lambda el: el != sentinel, iterable)<EOL>next(truncated)<EOL>return next(truncated)<EOL>

Get the value after `sentinel` in an `iterable`


pass<EOL>


session = settings.Session()<EOL>dms = session.query(DagModel).filter(<EOL>DagModel.dag_id.in_(DAG_IDS))<EOL>for dm in dms:<EOL><INDENT>logging.info('<STR_LIT>'.format(dm, is_paused))<EOL>dm.is_paused = is_paused<EOL><DEDENT>session.commit()<EOL>

Toggle the pause state of the DAGs in the test.


if not principal:<EOL><INDENT>return None<EOL><DEDENT>return re.split(r'<STR_LIT>', str(principal))<EOL>

get_components(principal) -> (short name, instance (FQDN), realm)

``principal`` is the kerberos principal to parse.


self._add_volume(name=volume.name, configs=volume.configs)<EOL>


self.volume_mounts.append({<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': mount_path,<EOL>'<STR_LIT>': sub_path,<EOL>'<STR_LIT>': read_only<EOL>})<EOL>

Args:
    name (str):
    mount_path (str):
    sub_path (str):
    read_only:

Returns:


def run_pod(self, pod, startup_timeout=<NUM_LIT>, get_logs=True):<EOL>

resp = self.run_pod_async(pod)<EOL>curr_time = dt.now()<EOL>if resp.status.start_time is None:<EOL><INDENT>while self.pod_not_started(pod):<EOL><INDENT>delta = dt.now() - curr_time<EOL>if delta.seconds >= startup_timeout:<EOL><INDENT>raise AirflowException("<STR_LIT>")<EOL><DEDENT>time.sleep(<NUM_LIT:1>)<EOL><DEDENT>self.log.debug('<STR_LIT>')<EOL><DEDENT>return self._monitor_pod(pod, get_logs)<EOL>

Launches the pod synchronously and waits for completion.
Args:
    pod (Pod):
    startup_timeout (int): Timeout for startup of the pod (if pod is pending for
     too long, considers task a failure


<EOL>if self.kube_config.dags_volume_claim orself.kube_config.dags_volume_host or self.kube_config.dags_in_image:<EOL><INDENT>return []<EOL><DEDENT>init_environment = [{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': self.kube_config.git_repo<EOL>}, {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': self.kube_config.git_branch<EOL>}, {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': self.kube_config.git_sync_root<EOL>}, {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': self.kube_config.git_sync_dest<EOL>}, {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT:1>'<EOL>}, {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT:true>'<EOL>}]<EOL>if self.kube_config.git_user:<EOL><INDENT>init_environment.append({<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': self.kube_config.git_user<EOL>})<EOL><DEDENT>if self.kube_config.git_password:<EOL><INDENT>init_environment.append({<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': self.kube_config.git_password<EOL>})<EOL><DEDENT>volume_mounts = [{<EOL>'<STR_LIT>': self.kube_config.git_sync_root,<EOL>'<STR_LIT:name>': self.dags_volume_name,<EOL>'<STR_LIT>': False<EOL>}]<EOL>if self.kube_config.git_ssh_key_secret_name:<EOL><INDENT>volume_mounts.append({<EOL>'<STR_LIT:name>': self.git_sync_ssh_secret_volume_name,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>})<EOL>init_environment.extend([<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT>'<EOL>},<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT:true>'<EOL>}])<EOL><DEDENT>if self.kube_config.git_ssh_known_hosts_configmap_name:<EOL><INDENT>volume_mounts.append({<EOL>'<STR_LIT:name>': self.git_sync_ssh_known_hosts_volume_name,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>})<EOL>init_environment.extend([<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT:true>'<EOL>},<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT>'<EOL>}<EOL>])<EOL><DEDENT>else:<EOL><INDENT>init_environment.append({<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT:false>'<EOL>})<EOL><DEDENT>return [{<EOL>'<STR_LIT:name>': self.kube_config.git_sync_init_container_name,<EOL>'<STR_LIT:image>': self.kube_config.git_sync_container,<EOL>'<STR_LIT>': {'<STR_LIT>': <NUM_LIT>},  <EOL>'<STR_LIT>': init_environment,<EOL>'<STR_LIT>': volume_mounts<EOL>}]<EOL>

When using git to retrieve the DAGs, use the GitSync Init Container


try:<EOL><INDENT>for validation_spec in self._validation_specs:<EOL><INDENT>self._validate_field(validation_spec=validation_spec,<EOL>dictionary_to_validate=body_to_validate)<EOL><DEDENT><DEDENT>except GcpFieldValidationException as e:<EOL><INDENT>raise GcpFieldValidationException(<EOL>"<STR_LIT>".<EOL>format(body_to_validate, e))<EOL><DEDENT>all_field_names = [spec['<STR_LIT:name>'] for spec in self._validation_specs<EOL>if spec.get('<STR_LIT:type>') != '<STR_LIT>' and<EOL>spec.get('<STR_LIT>') != self._api_version]<EOL>all_union_fields = [spec for spec in self._validation_specs<EOL>if spec.get('<STR_LIT:type>') == '<STR_LIT>']<EOL>for union_field in all_union_fields:<EOL><INDENT>all_field_names.extend(<EOL>[nested_union_spec['<STR_LIT:name>'] for nested_union_spec in union_field['<STR_LIT>']<EOL>if nested_union_spec.get('<STR_LIT:type>') != '<STR_LIT>' and<EOL>nested_union_spec.get('<STR_LIT>') != self._api_version])<EOL><DEDENT>for field_name in body_to_validate.keys():<EOL><INDENT>if field_name not in all_field_names:<EOL><INDENT>self.log.warning(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>",<EOL>field_name, self._validation_specs)<EOL><DEDENT><DEDENT>

Validates if the body (dictionary) follows specification that the validator was
instantiated with. Raises ValidationSpecificationException or
ValidationFieldException in case of problems with specification or the
body not conforming to the specification respectively.

:param body_to_validate: body that must follow the specification
:type body_to_validate: dict
:return: None


for channel in tuning_config['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>self.check_s3_url(channel['<STR_LIT>']['<STR_LIT>']['<STR_LIT>'])<EOL><DEDENT>

Check if a tuning configuration is valid

:param tuning_config: tuning_config
:type tuning_config: dict
:return: None


def multi_stream_iter(self, log_group, streams, positions=None):

positions = positions or {s: Position(timestamp=<NUM_LIT:0>, skip=<NUM_LIT:0>) for s in streams}<EOL>event_iters = [self.log_stream(log_group, s, positions[s].timestamp, positions[s].skip)<EOL>for s in streams]<EOL>events = []<EOL>for s in event_iters:<EOL><INDENT>if not s:<EOL><INDENT>events.append(None)<EOL>continue<EOL><DEDENT>try:<EOL><INDENT>events.append(next(s))<EOL><DEDENT>except StopIteration:<EOL><INDENT>events.append(None)<EOL><DEDENT><DEDENT>while any(events):<EOL><INDENT>i = argmin(events, lambda x: x['<STR_LIT>'] if x else <NUM_LIT>)<EOL>yield (i, events[i])<EOL>try:<EOL><INDENT>events[i] = next(event_iters[i])<EOL><DEDENT>except StopIteration:<EOL><INDENT>events[i] = None<EOL><DEDENT><DEDENT>

Iterate over the available events coming from a set of log streams in a single log group
interleaving the events from each stream so they're yielded in timestamp order.

:param log_group: The name of the log group.
:type log_group: str
:param streams: A list of the log stream names. The position of the stream in this list is
    the stream number.
:type streams: list
:param positions: A list of pairs of (timestamp, skip) which represents the last record
    read from each stream.
:type positions: list
:return: A tuple of (stream number, cloudwatch log event).


def check_status(self, job_name, key,<EOL>describe_function, check_interval,<EOL>max_ingestion_time,<EOL>non_terminal_states=None):

if not non_terminal_states:<EOL><INDENT>non_terminal_states = self.non_terminal_states<EOL><DEDENT>sec = <NUM_LIT:0><EOL>running = True<EOL>while running:<EOL><INDENT>time.sleep(check_interval)<EOL>sec = sec + check_interval<EOL>try:<EOL><INDENT>response = describe_function(job_name)<EOL>status = response[key]<EOL>self.log.info('<STR_LIT>'<EOL>'<STR_LIT>' % (sec, status))<EOL><DEDENT>except KeyError:<EOL><INDENT>raise AirflowException('<STR_LIT>')<EOL><DEDENT>except ClientError:<EOL><INDENT>raise AirflowException('<STR_LIT>')<EOL><DEDENT>if status in non_terminal_states:<EOL><INDENT>running = True<EOL><DEDENT>elif status in self.failed_states:<EOL><INDENT>raise AirflowException('<STR_LIT>' % response['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>running = False<EOL><DEDENT>if max_ingestion_time and sec > max_ingestion_time:<EOL><INDENT>raise AirflowException('<STR_LIT>', max_ingestion_time)<EOL><DEDENT><DEDENT>self.log.info('<STR_LIT>')<EOL>response = describe_function(job_name)<EOL>return response<EOL>

Check status of a SageMaker job

:param job_name: name of the job to check status
:type job_name: str
:param key: the key of the response dict
    that points to the state
:type key: str
:param describe_function: the function used to retrieve the status
:type describe_function: python callable
:param args: the arguments for the function
:param check_interval: the time interval in seconds which the operator
    will check the status of any SageMaker job
:type check_interval: int
:param max_ingestion_time: the maximum ingestion time in seconds. Any
    SageMaker jobs that run longer than this will fail. Setting this to
    None implies no timeout for any SageMaker job.
:type max_ingestion_time: int
:param non_terminal_states: the set of nonterminal states
:type non_terminal_states: set
:return: response of describe call after job is done


def create_tuning_job(self, config, wait_for_completion=True,<EOL>check_interval=<NUM_LIT:30>, max_ingestion_time=None):

Create a tuning job

:param config: the config for tuning
:type config: dict
:param wait_for_completion: if the program should keep running until job finishes
:type wait_for_completion: bool
:param check_interval: the time interval in seconds which the operator
    will check the status of any SageMaker job
:type check_interval: int
:param max_ingestion_time: the maximum ingestion time in seconds. Any
    SageMaker jobs that run longer than this will fail. Setting this to
    None implies no timeout for any SageMaker job.
:type max_ingestion_time: int
:return: A response to tuning job creation


return self.get_conn().describe_endpoint_config(EndpointConfigName=name)<EOL>

Return the endpoint config info associated with the name

:param name: the name of the endpoint config
:type name: string
:return: A dict contains all the endpoint config info


def create_cluster(self, cluster, project_id=None, retry=DEFAULT, timeout=DEFAULT):

if isinstance(cluster, dict):<EOL><INDENT>cluster_proto = Cluster()<EOL>cluster = self._dict_to_proto(py_dict=cluster, proto=cluster_proto)<EOL><DEDENT>elif not isinstance(cluster, Cluster):<EOL><INDENT>raise AirflowException(<EOL>"<STR_LIT>")<EOL><DEDENT>self._append_label(cluster, '<STR_LIT>', '<STR_LIT:v>' + version.version)<EOL>self.log.info(<EOL>"<STR_LIT>",<EOL>self.project_id, self.location, cluster.name<EOL>)<EOL>try:<EOL><INDENT>op = self.get_client().create_cluster(project_id=project_id or self.project_id,<EOL>zone=self.location,<EOL>cluster=cluster,<EOL>retry=retry,<EOL>timeout=timeout)<EOL>op = self.wait_for_operation(op)<EOL>return op.target_link<EOL><DEDENT>except AlreadyExists as error:<EOL><INDENT>self.log.info('<STR_LIT>', error.message)<EOL>return self.get_cluster(name=cluster.name).self_link<EOL><DEDENT>

Creates a cluster, consisting of the specified number and type of Google Compute
Engine instances.

:param cluster: A Cluster protobuf or dict. If dict is provided, it must
    be of the same form as the protobuf message
    :class:`google.cloud.container_v1.types.Cluster`
:type cluster: dict or google.cloud.container_v1.types.Cluster
:param project_id: Google Cloud Platform project ID
:type project_id: str
:param retry: A retry object (``google.api_core.retry.Retry``) used to
    retry requests.
    If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: The amount of time, in seconds, to wait for the request to
    complete. Note that if retry is specified, the timeout applies to each
    individual attempt.
:type timeout: float
:return: The full url to the new, or existing, cluster
:raises:
    ParseError: On JSON parsing problems when trying to convert dict
    AirflowException: cluster is not dict type nor Cluster proto type


def delete_subscription(self, project, subscription,<EOL>fail_if_not_exists=False):

service = self.get_conn()<EOL>full_subscription = _format_subscription(project, subscription)<EOL>try:<EOL><INDENT>service.projects().subscriptions().delete(<EOL>subscription=full_subscription).execute(num_retries=self.num_retries)<EOL><DEDENT>except HttpError as e:<EOL><INDENT>if str(e.resp['<STR_LIT:status>']) == '<STR_LIT>':<EOL><INDENT>message = '<STR_LIT>'.format(<EOL>full_subscription)<EOL>self.log.warning(message)<EOL>if fail_if_not_exists:<EOL><INDENT>raise PubSubException(message)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise PubSubException(<EOL>'<STR_LIT>'.format(full_subscription),<EOL>e)<EOL><DEDENT><DEDENT>

Deletes a Pub/Sub subscription, if it exists.

        :param project: the GCP project ID where the subscription exists
        :type project: str
        :param subscription: the Pub/Sub subscription name to delete; do not
            include the ``projects/{project}/subscriptions/`` prefix.
        :type subscription: str
        :param fail_if_not_exists: if set, raise an exception if the topic
            does not exist
        :type fail_if_not_exists: bool


def publish(self, project, topic, messages):

body = {'<STR_LIT>': messages}<EOL>full_topic = _format_topic(project, topic)<EOL>request = self.get_conn().projects().topics().publish(<EOL>topic=full_topic, body=body)<EOL>try:<EOL><INDENT>request.execute(num_retries=self.num_retries)<EOL><DEDENT>except HttpError as e:<EOL><INDENT>raise PubSubException(<EOL>'<STR_LIT>'.format(full_topic), e)<EOL><DEDENT>

Publishes messages to a Pub/Sub topic.

        :param project: the GCP project ID in which to publish
        :type project: str
        :param topic: the Pub/Sub topic to which to publish; do not
            include the ``projects/{project}/topics/`` prefix.
        :type topic: str
        :param messages: messages to publish; if the data field in a
            message is set, it should already be base64 encoded.
        :type messages: list of PubSub messages; see
            http://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage


def acknowledge(self, project, subscription, ack_ids):

service = self.get_conn()<EOL>full_subscription = _format_subscription(project, subscription)<EOL>try:<EOL><INDENT>service.projects().subscriptions().acknowledge(<EOL>subscription=full_subscription, body={'<STR_LIT>': ack_ids}<EOL>).execute(num_retries=self.num_retries)<EOL><DEDENT>except HttpError as e:<EOL><INDENT>raise PubSubException(<EOL>'<STR_LIT>'<EOL>.format(len(ack_ids), full_subscription), e)<EOL><DEDENT>

Pulls up to ``max_messages`` messages from Pub/Sub subscription.

        :param project: the GCP project name or ID in which to create
            the topic
        :type project: str
        :param subscription: the Pub/Sub subscription name to delete; do not
            include the 'projects/{project}/topics/' prefix.
        :type subscription: str
        :param ack_ids: List of ReceivedMessage ackIds from a previous pull
            response
        :type ack_ids: list


http_authorized = self._authorize()<EOL>return build(<EOL>'<STR_LIT>', '<STR_LIT>', http=http_authorized, cache_discovery=False)<EOL>

Returns a Pub/Sub service object.

        :rtype: googleapiclient.discovery.Resource


return project_id if project_id else self.project_id<EOL>

In case project_id is None, overrides it with default project_id from
the service account that is authorized.

:param project_id: project id to
:type project_id: str
:return: the project_id specified or default project id if project_id is None