def retype_path(<EOL>src, pyi_dir, targets, *, src_explicitly_given=False, quiet=False, hg=False<EOL>):

if src.is_dir():<EOL><INDENT>for child in src.iterdir():<EOL><INDENT>if child == pyi_dir or child == targets:<EOL><INDENT>continue<EOL><DEDENT>yield from retype_path(<EOL>child, pyi_dir / src.name, targets / src.name, quiet=quiet, hg=hg,<EOL>)<EOL><DEDENT><DEDENT>elif src.suffix == '<STR_LIT>' or src_explicitly_given:<EOL><INDENT>try:<EOL><INDENT>retype_file(src, pyi_dir, targets, quiet=quiet, hg=hg)<EOL><DEDENT>except Exception as e:<EOL><INDENT>yield (<EOL>src,<EOL>str(e),<EOL>type(e),<EOL>traceback.format_tb(e.__traceback__),<EOL>)<EOL><DEDENT><DEDENT>

Recursively retype files or directories given. Generate errors.


def retype_file(src, pyi_dir, targets, *, quiet=False, hg=False):

with tokenize.open(src) as src_buffer:<EOL><INDENT>src_encoding = src_buffer.encoding<EOL>src_node = lib2to3_parse(src_buffer.read())<EOL><DEDENT>try:<EOL><INDENT>with open((pyi_dir / src.name).with_suffix('<STR_LIT>')) as pyi_file:<EOL><INDENT>pyi_txt = pyi_file.read()<EOL><DEDENT><DEDENT>except FileNotFoundError:<EOL><INDENT>if not quiet:<EOL><INDENT>print(<EOL>f'<STR_LIT>',<EOL>file=sys.stderr,<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>pyi_ast = ast3.parse(pyi_txt)<EOL>assert isinstance(pyi_ast, ast3.Module)<EOL>reapply_all(pyi_ast.body, src_node)<EOL><DEDENT>fix_remaining_type_comments(src_node)<EOL>targets.mkdir(parents=True, exist_ok=True)<EOL>with open(targets / src.name, '<STR_LIT:w>', encoding=src_encoding) as target_file:<EOL><INDENT>target_file.write(lib2to3_unparse(src_node, hg=hg))<EOL><DEDENT>return targets / src.name<EOL>

Retype `src`, finding types in `pyi_dir`. Save in `targets`.

    The file should remain formatted exactly as it was before, save for:
    - annotations
    - additional imports needed to satisfy annotations
    - additional module-level names needed to satisfy annotations

    Type comments in sources are normalized to type annotations.


grammar = pygram.python_grammar_no_print_statement<EOL>drv = driver.Driver(grammar, pytree.convert)<EOL>if src_txt[-<NUM_LIT:1>] != '<STR_LIT:\n>':<EOL><INDENT>nl = '<STR_LIT:\r\n>' if '<STR_LIT:\r\n>' in src_txt[:<NUM_LIT>] else '<STR_LIT:\n>'<EOL>src_txt += nl<EOL><DEDENT>try:<EOL><INDENT>result = drv.parse_string(src_txt, True)<EOL><DEDENT>except ParseError as pe:<EOL><INDENT>lineno, column = pe.context[<NUM_LIT:1>]<EOL>lines = src_txt.splitlines()<EOL>try:<EOL><INDENT>faulty_line = lines[lineno - <NUM_LIT:1>]<EOL><DEDENT>except IndexError:<EOL><INDENT>faulty_line = "<STR_LIT>"<EOL><DEDENT>raise ValueError(f"<STR_LIT>") from None<EOL><DEDENT>if isinstance(result, Leaf):<EOL><INDENT>result = Node(syms.file_input, [result])<EOL><DEDENT>return result<EOL>

Given a string with source, return the lib2to3 Node.


code = str(node)<EOL>if hg:<EOL><INDENT>from retype_hgext import apply_job_security<EOL>code = apply_job_security(code)<EOL><DEDENT>return code<EOL>

Given a lib2to3 node, return its string representation.


late_processing = reapply(ast_node, lib2to3_node)<EOL>for lazy_func in reversed(late_processing):<EOL><INDENT>lazy_func()<EOL><DEDENT>

Reapplies the typed_ast node into the lib2to3 tree.

    Also does post-processing. This is done in reverse order to enable placing
    TypeVars and aliases that depend on one another.


return []<EOL>

Reapplies the typed_ast node into the lib2to3 tree.

    By default does nothing.


return "<STR_LIT>"<EOL>

serialize_attribute(Attribute()) -> "self.f1.f2.f3"

    Change an AST object into its string representation.


raise NotImplementedError(f"<STR_LIT>")<EOL>

Converts an AST object into its lib2to3 equivalent.


return False<EOL>

Returns True if `node` represents `names`.


assert node.type == syms.file_input<EOL>last_n = None<EOL>for n in node.post_order():<EOL><INDENT>if last_n is not None:<EOL><INDENT>if n.type == token.NEWLINE and is_assignment(last_n):<EOL><INDENT>fix_variable_annotation_type_comment(n, last_n)<EOL><DEDENT>elif n.type == syms.funcdef and last_n.type == syms.suite:<EOL><INDENT>fix_signature_annotation_type_comment(n, last_n, offset=<NUM_LIT:1>)<EOL><DEDENT>elif n.type == syms.async_funcdef and last_n.type == syms.suite:<EOL><INDENT>fix_signature_annotation_type_comment(n, last_n, offset=<NUM_LIT:2>)<EOL><DEDENT><DEDENT>last_n = n<EOL><DEDENT>

Converts type comments in `node` to proper annotated assignments.


def get_function_signature(fun, *, is_method=False):

args = fun.args<EOL>returns = fun.returns<EOL>if fun.type_comment:<EOL><INDENT>try:<EOL><INDENT>args_tc, returns_tc = parse_signature_type_comment(fun.type_comment)<EOL>if returns and returns_tc:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>returns = returns_tc<EOL>copy_arguments_to_annotations(args, args_tc, is_method=is_method)<EOL><DEDENT>except (SyntaxError, ValueError) as exc:<EOL><INDENT>raise ValueError(<EOL>f"<STR_LIT>" +<EOL>f"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>copy_type_comments_to_annotations(args)<EOL>return args, returns<EOL>

Returns (args, returns).

    `args` is ast3.arguments, `returns` is the return type AST node. The kicker
    about this function is that it pushes type comments into proper annotation
    fields, standardizing type handling.


try:<EOL><INDENT>result = ast3.parse(type_comment, '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>except SyntaxError:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>assert isinstance(result, ast3.FunctionType)<EOL>if len(result.argtypes) == <NUM_LIT:1>:<EOL><INDENT>argtypes = result.argtypes[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>argtypes = result.argtypes<EOL><DEDENT>return argtypes, result.returns<EOL>

Parse the fugly signature type comment into AST nodes.

    Caveats: ASTifying **kwargs is impossible with the current grammar so we
    hack it into unary subtraction (to differentiate from Starred in vararg).

    For example from:
    "(str, int, *int, **Any) -> 'SomeReturnType'"

    To:
    ([ast3.Name, ast.Name, ast3.Name, ast.Name], ast3.Str)


try:<EOL><INDENT>result = ast3.parse(type_comment, '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>except SyntaxError:<EOL><INDENT>raise ValueError(f"<STR_LIT>") from None<EOL><DEDENT>assert isinstance(result, ast3.Expression)<EOL>return result.body<EOL>

Parse a type comment string into AST nodes.


arguments = f"<STR_LIT>"<EOL>try:<EOL><INDENT>result = ast3.parse(arguments, '<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>except SyntaxError:<EOL><INDENT>raise ValueError(f"<STR_LIT>") from None<EOL><DEDENT>assert isinstance(result, ast3.Module)<EOL>assert len(result.body) == <NUM_LIT:1><EOL>assert isinstance(result.body[<NUM_LIT:0>], ast3.FunctionDef)<EOL>args = result.body[<NUM_LIT:0>].args<EOL>copy_type_comments_to_annotations(args)<EOL>return args<EOL>

parse_arguments('(a, b, *, c=False, **d)') -> ast3.arguments

    Parse a string with function arguments into an AST node.


def copy_arguments_to_annotations(args, type_comment, *, is_method=False):

if isinstance(type_comment, ast3.Ellipsis):<EOL><INDENT>return<EOL><DEDENT>expected = len(args.args)<EOL>if args.vararg:<EOL><INDENT>expected += <NUM_LIT:1><EOL><DEDENT>expected += len(args.kwonlyargs)<EOL>if args.kwarg:<EOL><INDENT>expected += <NUM_LIT:1><EOL><DEDENT>actual = len(type_comment) if isinstance(type_comment, list) else <NUM_LIT:1><EOL>if expected != actual:<EOL><INDENT>if is_method and expected - actual == <NUM_LIT:1>:<EOL><INDENT>pass  <EOL><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>f"<STR_LIT>" +<EOL>f"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>if isinstance(type_comment, list):<EOL><INDENT>next_value = type_comment.pop<EOL><DEDENT>else:<EOL><INDENT>_tc = type_comment<EOL>def next_value(index: int = <NUM_LIT:0>) -> ast3.expr:<EOL><INDENT>return _tc<EOL><DEDENT><DEDENT>for arg in args.args[expected - actual:]:<EOL><INDENT>ensure_no_annotation(arg.annotation)<EOL>arg.annotation = next_value(<NUM_LIT:0>)<EOL><DEDENT>if args.vararg:<EOL><INDENT>ensure_no_annotation(args.vararg.annotation)<EOL>args.vararg.annotation = next_value(<NUM_LIT:0>)<EOL><DEDENT>for arg in args.kwonlyargs:<EOL><INDENT>ensure_no_annotation(arg.annotation)<EOL>arg.annotation = next_value(<NUM_LIT:0>)<EOL><DEDENT>if args.kwarg:<EOL><INDENT>ensure_no_annotation(args.kwarg.annotation)<EOL>args.kwarg.annotation = next_value(<NUM_LIT:0>)<EOL><DEDENT>

Copies AST nodes from `type_comment` into the ast3.arguments in `args`.

    Does validaation of argument count (allowing for untyped self/cls)
    and type (vararg and kwarg).


for arg in args.args:<EOL><INDENT>copy_type_comment_to_annotation(arg)<EOL><DEDENT>if args.vararg:<EOL><INDENT>copy_type_comment_to_annotation(args.vararg)<EOL><DEDENT>for arg in args.kwonlyargs:<EOL><INDENT>copy_type_comment_to_annotation(arg)<EOL><DEDENT>if args.kwarg:<EOL><INDENT>copy_type_comment_to_annotation(args.kwarg)<EOL><DEDENT>

Copies argument type comments from the legacy long form to annotations
    in the entire function signature.


def maybe_replace_any_if_equal(name, expected, actual):

is_equal = expected == actual<EOL>if not is_equal and Config.replace_any:<EOL><INDENT>actual_str = minimize_whitespace(str(actual))<EOL>if actual_str and actual_str[<NUM_LIT:0>] in {'<STR_LIT:">', "<STR_LIT:'>"}:<EOL><INDENT>actual_str = actual_str[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL><DEDENT>is_equal = actual_str in {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'}<EOL><DEDENT>if not is_equal:<EOL><INDENT>expected_annotation = minimize_whitespace(str(expected))<EOL>actual_annotation = minimize_whitespace(str(actual))<EOL>raise ValueError(<EOL>f"<STR_LIT>" +<EOL>f"<STR_LIT>"<EOL>)<EOL><DEDENT>return expected or actual<EOL>

Return the type given in `expected`.

    Raise ValueError if `expected` isn't equal to `actual`.  If --replace-any is
    used, the Any type in `actual` is considered equal.

    The implementation is naively checking if the string representation of
    `actual` is one of "Any", "typing.Any", or "t.Any".  This is done for two
    reasons:

    1. I'm lazy.
    2. We want people to be able to explicitly state that they want Any without it
       being replaced.  This way they can use an alias.


def ensure_annotations_equal(name, expected, actual):

maybe_replace_any_if_equal(name, expected, actual)<EOL>

Raise ValueError if `expected` isn't equal to `actual`.

    If --replace-any is used, the Any type in `actual` is considered equal.


for node in body.children:<EOL><INDENT>if node.type == token.INDENT:<EOL><INDENT>prefix = node.prefix.lstrip()<EOL>if prefix.startswith('<STR_LIT>'):<EOL><INDENT>node.prefix = '<STR_LIT:\n>'.join(prefix.split('<STR_LIT:\n>')[<NUM_LIT:1>:])<EOL><DEDENT>break<EOL><DEDENT><DEDENT>

Removes the legacy signature type comment, leaving other comments if any.


for node in children:<EOL><INDENT>if node.type in (syms.try_stmt, syms.suite):<EOL><INDENT>yield from flatten_some(node.children)<EOL><DEDENT>else:<EOL><INDENT>yield node<EOL><DEDENT><DEDENT>

Generates nodes or leaves, unpacking bodies of try:except:finally: statements.


default = None<EOL>name = params.pop(<NUM_LIT:0>)<EOL>if name in (_star, _dstar):<EOL><INDENT>default = params.pop(<NUM_LIT:0>)<EOL>if default == _comma:<EOL><INDENT>return name, default<EOL><DEDENT><DEDENT>try:<EOL><INDENT>remainder = params.pop(<NUM_LIT:0>)<EOL>if remainder == _eq:<EOL><INDENT>default = params.pop(<NUM_LIT:0>)<EOL>remainder = params.pop(<NUM_LIT:0>)<EOL><DEDENT>if remainder != _comma:<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT><DEDENT>except IndexError:<EOL><INDENT>pass<EOL><DEDENT>return name, default<EOL>

Pops the parameter and the "remainder" (comma, default value).

    Returns a tuple of ('name', default) or (_star, 'name') or (_dstar, 'name').


assert body.type in (syms.file_input, syms.suite)<EOL>_offset = <NUM_LIT:0><EOL>prefix = '<STR_LIT>'<EOL>for _offset, child in enumerate(body.children):<EOL><INDENT>if child.type == syms.simple_stmt:<EOL><INDENT>stmt = child.children[<NUM_LIT:0>]<EOL>if stmt.type == syms.expr_stmt:<EOL><INDENT>expr = stmt.children<EOL>if not skip_assignments:<EOL><INDENT>break<EOL><DEDENT>if (<EOL>len(expr) != <NUM_LIT:2> or<EOL>expr[<NUM_LIT:0>].type != token.NAME or<EOL>expr[<NUM_LIT:1>].type != syms.annassign or<EOL>_eq in expr[<NUM_LIT:1>].children<EOL>):<EOL><INDENT>break<EOL><DEDENT><DEDENT>elif stmt.type not in (syms.import_name, syms.import_from, token.STRING):<EOL><INDENT>break<EOL><DEDENT><DEDENT>elif child.type == token.INDENT:<EOL><INDENT>assert isinstance(child, Leaf)<EOL>prefix = child.value<EOL><DEDENT>elif child.type != token.NEWLINE:<EOL><INDENT>break<EOL><DEDENT><DEDENT>prefix, child.prefix = child.prefix, prefix<EOL>return _offset, prefix<EOL>

Returns the offset after which a statement can be inserted to the `body`.

    This offset is calculated to come after all imports, and maybe existing
    (possibly annotated) assignments if `skip_assignments` is True.

    Also returns the indentation prefix that should be applied to the inserted
    node.


Returns True if `name` appears in `node`. False otherwise.


maxline = <NUM_LIT:0><EOL>for node in body.pre_order():<EOL><INDENT>maxline += node.prefix.count('<STR_LIT:\n>')<EOL>if isinstance(node, Leaf):<EOL><INDENT>node.lineno = maxline<EOL>maxline += str(node.value).count('<STR_LIT:\n>')<EOL><DEDENT><DEDENT>

r"""Recomputes all line numbers based on the number of \n characters.


if isinstance(n, Leaf):<EOL><INDENT>return Leaf(n.type, n.value, prefix=n.prefix if prefix is None else prefix)<EOL><DEDENT>n.parent = None<EOL>if prefix is not None:<EOL><INDENT>n.prefix = prefix<EOL><DEDENT>return n<EOL>

lib2to3's AST requires unique objects as children.


buf = io.BytesIO(code.encode('<STR_LIT:utf8>'))<EOL>tokens = tokenize.tokenize(buf.readline)<EOL>data = tokenize.untokenize(replacetokens(list(tokens), '<STR_LIT>'))<EOL>return cast(str, data.decode('<STR_LIT:utf8>'))<EOL>

Treat input `code` like Python 2 (implicit strings are byte literals).

    The implementation is horribly inefficient but the goal is to be compatible
    with what Mercurial does at runtime.


def create_index(file_path, index_path, index_ratio, index_width):

i = <NUM_LIT:0><EOL>with file_path.open() as f:<EOL><INDENT>with index_path.open("<STR_LIT:wb>") as idx:<EOL><INDENT>idx.write(index_ratio.to_bytes(<NUM_LIT:1>, byteorder="<STR_LIT>"))<EOL>idx.write(index_width.to_bytes(<NUM_LIT:1>, byteorder="<STR_LIT>"))<EOL>idx.write((<NUM_LIT:0>).to_bytes(<NUM_LIT:32>, byteorder="<STR_LIT>"))  <EOL>idx.write((<NUM_LIT:0>).to_bytes(index_width, byteorder="<STR_LIT>"))<EOL>while f.readline():<EOL><INDENT>i += <NUM_LIT:1><EOL>if (i % index_ratio) == <NUM_LIT:0>:<EOL><INDENT>pointer = f.tell()<EOL>b = pointer.to_bytes(index_width, byteorder="<STR_LIT>")<EOL>idx.write(b)<EOL><DEDENT><DEDENT>idx.seek(<NUM_LIT:2>)<EOL>idx.write(i.to_bytes(<NUM_LIT:32>, byteorder="<STR_LIT>"))<EOL><DEDENT>t = file_path.stat().st_mtime<EOL>os.utime(str(index_path), (t, t))<EOL><DEDENT>

Index format:
    1st byte: index_ratio
    2nd byte: index_width
    3rd byte: line_count


def __init__(self, filepath, encoding=None, errors=None, newline=None, index_ratio=<NUM_LIT:1>, index_width=<NUM_LIT:5>):

self.filepath = Path(filepath)<EOL>self.file = self.filepath.open(encoding=encoding, errors=errors, newline=newline)<EOL>self.index = self.get_or_create_index(index_ratio, index_width)<EOL>

Open a text file which has it's line numbers indexed.

:param filepath:
:param encoding:
:param errors:
:param newline:
:param index_ratio:  how close together should indexed lines be? (1=index all lines, 2=index every 2nd line, etc)
:param index_width: how many bytes are needed to store the location of lines in the file


start, stop, step = normalize_slice(slice_obj, self.index.line_count)<EOL>if isinstance(slice_obj, slice):<EOL><INDENT>if step == <NUM_LIT:1>:<EOL><INDENT>return self._get_lines(start, stop)<EOL><DEDENT>return [self._get_lines(i)[<NUM_LIT:0>] for i in range(start, stop, step)]<EOL><DEDENT>if isinstance(slice_obj, int):<EOL><INDENT>return self._get_lines(start)[<NUM_LIT:0>]<EOL><DEDENT>return None<EOL>

Supports slice operations on the file

For example:

    with IndexedOpen(filename) as f:
        print f[6:-2]


def get_or_create_index(self, index_ratio, index_width):

if not self.index_path.exists() or not self.filepath.stat().st_mtime == self.index_path.stat().st_mtime:<EOL><INDENT>create_index(self.filepath, self.index_path, index_ratio=index_ratio, index_width=index_width)<EOL><DEDENT>return IndexFile(str(self.index_path))<EOL>

Return an open file-object to the index file


return Path(str(self.filepath) + "<STR_LIT>")<EOL>


self.index_path = Path(index_file_path)<EOL>self.header_length = <NUM_LIT><EOL>self.index_file = self._open_index()<EOL>

Open a line-number index for a text file.


if isinstance(slice_obj, slice):<EOL><INDENT>start, stop, step = slice_obj.start, slice_obj.stop, slice_obj.step<EOL>if start is None:<EOL><INDENT>start = <NUM_LIT:0><EOL><DEDENT>if stop is None:<EOL><INDENT>stop = length<EOL><DEDENT>if step is None:<EOL><INDENT>step = <NUM_LIT:1><EOL><DEDENT>if start < <NUM_LIT:0>:<EOL><INDENT>start += length<EOL><DEDENT>if stop < <NUM_LIT:0>:<EOL><INDENT>stop += length<EOL><DEDENT><DEDENT>elif isinstance(slice_obj, int):<EOL><INDENT>start = slice_obj<EOL>if start < <NUM_LIT:0>:<EOL><INDENT>start += length<EOL><DEDENT>stop = start + <NUM_LIT:1><EOL>step = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>raise TypeError<EOL><DEDENT>if (<NUM_LIT:0> <= start <= length) and (<NUM_LIT:0> <= stop <= length):<EOL><INDENT>return start, stop, step<EOL><DEDENT>raise IndexError<EOL>

Given a slice object, return appropriate values for use in the range function

:param slice_obj: The slice object or integer provided in the `[]` notation
:param length: For negative indexing we need to know the max length of the object.


global engine<EOL>engine = create_engine('<STR_LIT>', echo=False)<EOL>global Session<EOL>Session = sessionmaker(bind=engine)<EOL>global session<EOL>session = Session()<EOL>session._model_changes = {}<EOL>Base.metadata.create_all(bind=engine)<EOL>session.add_all([<EOL>User(name='<STR_LIT>', lastname='<STR_LIT>', uid='<STR_LIT>', city_id=<NUM_LIT:1>),<EOL>User(name='<STR_LIT>', lastname='<STR_LIT>', uid='<STR_LIT>', city_id=<NUM_LIT:2>),<EOL>User(name='<STR_LIT>', lastname='<STR_LIT>', uid='<STR_LIT>', city_id=<NUM_LIT:1>),<EOL>City(name='<STR_LIT>'),<EOL>City(name='<STR_LIT>')<EOL>])<EOL>session.commit()<EOL>


def elastic_query(model, query, session=None, enabled_fields=None):

<EOL>instance = ElasticQuery(model, query, session, enabled_fields)<EOL>return instance.search()<EOL>

Public method for init the class ElasticQuery
        :model: SQLAlchemy model
        :query: valid string like a ElasticSearch
        :session: SQLAlchemy session *optional
        :enabled_fields: Fields allowed for make a query *optional


def __init__(self, model, query, session=None, enabled_fields=None):

self.model = model<EOL>self.query = query<EOL>if hasattr(model, '<STR_LIT>'):<EOL><INDENT>self.model_query = model.query<EOL><DEDENT>else:<EOL><INDENT>self.model_query = session.query(self.model)<EOL><DEDENT>self.enabled_fields = enabled_fields<EOL>

Initializator of the class 'ElasticQuery


try:<EOL><INDENT>filters = json.loads(self.query)<EOL><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>result = self.model_query<EOL>if '<STR_LIT>'in filters.keys():<EOL><INDENT>result = self.parse_filter(filters['<STR_LIT>'])<EOL><DEDENT>if '<STR_LIT>'in filters.keys():<EOL><INDENT>result = result.order_by(*self.sort(filters['<STR_LIT>']))<EOL><DEDENT>return result<EOL>


for filter_type in filters:<EOL><INDENT>if filter_type == '<STR_LIT>' or filter_type == '<STR_LIT>':<EOL><INDENT>conditions = []<EOL>for field in filters[filter_type]:<EOL><INDENT>if self.is_field_allowed(field):<EOL><INDENT>conditions.append(self.create_query(self.parse_field(field, filters[filter_type][field])))<EOL><DEDENT><DEDENT>if filter_type == '<STR_LIT>':<EOL><INDENT>self.model_query = self.model_query.filter(or_(*conditions))<EOL><DEDENT>elif filter_type == '<STR_LIT>':<EOL><INDENT>self.model_query = self.model_query.filter(and_(*conditions))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.is_field_allowed(filter_type):<EOL><INDENT>conditions = self.create_query(self.parse_field(filter_type, filters[filter_type]))<EOL>self.model_query = self.model_query.filter(conditions)<EOL><DEDENT><DEDENT><DEDENT>return self.model_query<EOL>


def parse_field(self, field, field_value):

if type(field_value) is dict:<EOL><INDENT>operator = list(field_value)[<NUM_LIT:0>]<EOL>if self.verify_operator(operator) is False:<EOL><INDENT>return "<STR_LIT>", operator<EOL><DEDENT>value = field_value[operator]<EOL><DEDENT>elif type(field_value) is unicode:<EOL><INDENT>operator = u'<STR_LIT>'<EOL>value = field_value<EOL><DEDENT>return field, operator, value<EOL>

Parse the operators and traduce: ES to SQLAlchemy operators


try:<EOL><INDENT>if hasattr(OPERATORS[operator], '<STR_LIT>'):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>


field = attr[<NUM_LIT:0>]<EOL>operator = attr[<NUM_LIT:1>]<EOL>value = attr[<NUM_LIT:2>]<EOL>model = self.model<EOL>if '<STR_LIT:.>' in field:<EOL><INDENT>field_items = field.split('<STR_LIT:.>')<EOL>field_name = getattr(model, field_items[<NUM_LIT:0>], None)<EOL>class_name = field_name.property.mapper.class_<EOL>new_model = getattr(class_name, field_items[<NUM_LIT:1>])<EOL>return field_name.has(OPERATORS[operator](new_model, value))<EOL><DEDENT>return OPERATORS[operator](getattr(model, field, None), value)<EOL>


order = []<EOL>for sort in sort_list:<EOL><INDENT>if sort_list[sort] == "<STR_LIT>":<EOL><INDENT>order.append(asc(getattr(self.model, sort, None)))<EOL><DEDENT>elif sort_list[sort] == "<STR_LIT>":<EOL><INDENT>order.append(desc(getattr(self.model, sort, None)))<EOL><DEDENT><DEDENT>return order<EOL>


with open(os.path.join(*paths), '<STR_LIT:r>') as f:<EOL><INDENT>return f.read()<EOL><DEDENT>

Build a file path from *paths* and return the contents.


return u'<STR_LIT>'.join(iterable)<EOL>

A utility function to wrap up the converting a list of characters back into a string.


return string(zip(s.lower(), s.upper()))<EOL>

Attempts to match input to the letters in the string, without regard for case.


whitespace()<EOL>v = parser()<EOL>whitespace()<EOL>return v<EOL>

Ignores any whitespace surrounding parser.


quote_char = quote()<EOL>value, _ = many_until(parser, partial(one_of, quote_char))<EOL>return build_string(value)<EOL>

Parses as much as possible until it encounters a matching closing quote.

    By default matches any_token, but can be provided with a more specific parser if required.
    Returns a string


return partial(s, tri(string), s)<EOL>


return make_literal(s)()<EOL>


return partial(s, tri(caseless_string), s)<EOL>

returns a literal string, case independant parser.


return make_caseless_literal(s)()<EOL>


return lambda *args, **kwargs: f(g(*args, **kwargs))<EOL>

Compose returns a two functions composed as a new function.

    The first is called with the result of the second as its argument. Any arguments
    are passed to the second.


def chain_block(*args, **kwargs):<EOL><INDENT>v = args[<NUM_LIT:0>](*args, **kwargs)<EOL>for p in args[<NUM_LIT:1>:]:<EOL><INDENT>v = p(v)<EOL><DEDENT>return v<EOL><DEDENT>return chain_block<EOL>

Runs a series of parsers in sequence passing the result of each parser to the next.
    The result of the last parser is returned.


ch = peek()<EOL>if ch is EndOfFile:<EOL><INDENT>fail(["<STR_LIT>"])<EOL><DEDENT>next()<EOL>return ch<EOL>

Will accept any token in the input stream and step forward.

    Fails if there is no more tokens


ch = peek()<EOL>try:<EOL><INDENT>if (ch is EndOfFile) or (ch not in these):<EOL><INDENT>fail(list(these))<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>if ch != these:<EOL><INDENT>fail([these])<EOL><DEDENT><DEDENT>next()<EOL>return ch<EOL>

Returns the current token if is found in the collection provided.

    Fails otherwise.


ch = peek()<EOL>desc = "<STR_LIT>" + repr(these)<EOL>try:<EOL><INDENT>if (ch is EndOfFile) or (ch in these):<EOL><INDENT>fail([desc])<EOL><DEDENT><DEDENT>except TypeError:<EOL><INDENT>if ch != these:<EOL><INDENT>fail([desc])<EOL><DEDENT><DEDENT>next()<EOL>return ch<EOL>

Returns the current token if it is not found in the collection provided.

    The negative of one_of.


i = peek()<EOL>if (i is EndOfFile) or (not guard(i)):<EOL><INDENT>fail(["<STR_LIT>" + _fun_to_str(guard) + "<STR_LIT:>>"])<EOL><DEDENT>next()<EOL>return i<EOL>

Returns the current token if it satisfies the guard function provided.

    Fails otherwise.
    This is the a generalisation of one_of.


return choice(parser, lambda: default)<EOL>

Tries to apply the provided parser, returning default if the parser fails.


@tri<EOL>def not_followed_by_block():<EOL><INDENT>failed = object()<EOL>result = optional(tri(parser), failed)<EOL>if result != failed:<EOL><INDENT>fail(["<STR_LIT>" + _fun_to_str(parser)])<EOL><DEDENT><DEDENT>choice(not_followed_by_block)<EOL>

Succeeds if the given parser cannot consume input


results = []<EOL>terminate = object()<EOL>while local_ps.value:<EOL><INDENT>result = optional(parser, terminate)<EOL>if result == terminate:<EOL><INDENT>break<EOL><DEDENT>results.append(result)<EOL><DEDENT>return results<EOL>

Applies the parser to input zero or more times.

    Returns a list of parser results.


return [parser()] + many(parser)<EOL>

Like many, but must consume at least one of parser


results = []<EOL>while True:<EOL><INDENT>stop, result = choice(_tag(True, term),<EOL>_tag(False, these))<EOL>if stop:<EOL><INDENT>return results, result<EOL><DEDENT>else:<EOL><INDENT>results.append(result)<EOL><DEDENT><DEDENT>

Consumes as many of these as it can until it term is encountered.

    Returns a tuple of the list of these results and the term result


first = [these()]<EOL>these_results, term_result = many_until(these, term)<EOL>return (first + these_results, term_result)<EOL>

Like many_until but must consume at least one of these.


first = [parser()]<EOL>def inner():<EOL><INDENT>separator()<EOL>return parser()<EOL><DEDENT>return first + many(tri(inner))<EOL>

Like sep but must consume at least one of parser.


return optional(partial(sep1, parser, separator), [])<EOL>

Consumes zero or more of parser, separated by separator.

    Returns a list of parser's results


return [parser() for i in range(n)]<EOL>

Consumes n of parser, returning a list of the results.


found = []<EOL>for c in string:<EOL><INDENT>found.append(one_of(c))<EOL><DEDENT>return found<EOL>

Iterates over string, matching input to the items provided.

    The most obvious usage of this is to accept an entire string of characters,
    However this is function is more general than that. It takes an iterable
    and for each item, it tries one_of for that set. For example,
       string(['aA','bB','cC'])
    will accept 'abc' in either case.

    note, If you wish to match caseless strings as in the example, use
    picoparse.text.caseless_string.


if not parsers:<EOL><INDENT>return None<EOL><DEDENT>for p in parsers[:-<NUM_LIT:1>]:<EOL><INDENT>p()<EOL><DEDENT>return parsers[-<NUM_LIT:1>]()<EOL>

Runs multiple parsers and returns the result of the last.


if not parsers:<EOL><INDENT>return None<EOL><DEDENT>v = parsers[<NUM_LIT:0>]()<EOL>for p in parsers[<NUM_LIT:1>:]:<EOL><INDENT>p()<EOL><DEDENT>return v<EOL>

Runs multiple parsers and returns the result of the first.


tokens = []<EOL>while peek() is not EndOfFile:<EOL><INDENT>tokens.append(peek())<EOL>next()<EOL><DEDENT>return tokens<EOL>

Returns the remaining input that has not been parsed.


results = {}<EOL>for p in sequence:<EOL><INDENT>if callable(p): <EOL><INDENT>p()<EOL>continue<EOL><DEDENT>k, v = p<EOL>results[k] = v()<EOL><DEDENT>return results<EOL>

Runs a series of parsers in sequence optionally storing results in a returned dictionary.

    For example:
    seq(whitespace, ('phone', digits), whitespace, ('name', remaining))


try:<EOL><INDENT>for i in range(size):<EOL><INDENT>self.buffer.append(self.source.__next__())<EOL><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>self.buffer.append((EndOfFile, EndOfFile))<EOL><DEDENT>self.len = len(self.buffer)<EOL>

fills the internal buffer from the source iterator


self.index += <NUM_LIT:1><EOL>t = self.peek()<EOL>if not self.depth:<EOL><INDENT>self._cut()<EOL><DEDENT>return t<EOL>

Advances to and returns the next token or returns EndOfFile


if self.index >= self.len:<EOL><INDENT>self._fill((self.index - self.len) + <NUM_LIT:1>)<EOL><DEDENT>return self.index < self.len and self.buffer[self.index] or (EndOfFile, EndOfFile)<EOL>

Returns the current (token, position) or (EndOfFile, EndOfFile)


return self.current()[<NUM_LIT:0>]<EOL>


return self.current()[<NUM_LIT:1>]<EOL>

Returns the current position or EndOfFile


expec = asarray(expec, float)<EOL>if expec.ndim != <NUM_LIT:2>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>ploidy = expec.shape[-<NUM_LIT:1>]<EOL>return expec.sum(-<NUM_LIT:2>) / ploidy<EOL>

r""" Compute allele frequency from its expectation.

    Parameters
    ----------
    expec : array_like
        Allele expectations encoded as a samples-by-alleles matrix.

    Returns
    -------
    :class:`numpy.ndarray`
        Allele frequencies encoded as a variants-by-alleles matrix.

    Examples
    --------
    .. doctest::

        >>> from bgen_reader import read_bgen, example_files
        >>> from bgen_reader import allele_expectation, allele_frequency
        >>>
        >>> # Download an example
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> bgen = read_bgen(filepath, verbose=False)
        >>>
        >>> variants = bgen["variants"]
        >>> samples = bgen["samples"]
        >>> genotype = bgen["genotype"]
        >>>
        >>> variant = variants[variants["rsid"] == "RSID_6"].compute()
        >>> variant_idx = variant.index.item()
        >>>
        >>> p = genotype[variant_idx].compute()["probs"]
        >>> # For unphased genotypes only.
        >>> e = allele_expectation(bgen, variant_idx)
        >>> f = allele_frequency(e)
        >>>
        >>> alleles = variant["allele_ids"].item().split(",")
        >>> print(alleles[0] + ": {}".format(f[0]))
        A: 229.23103218810434
        >>> print(alleles[1] + ": {}".format(f[1]))
        G: 270.7689678118956
        >>> print(variant)
                id    rsid chrom   pos  nalleles allele_ids  vaddr
        4  SNPID_6  RSID_6    01  6000         2        A,G  19377
        >>>
        >>> # Clean-up the example
        >>> example.close()


if alt is None:<EOL><INDENT>return expec[..., -<NUM_LIT:1>]<EOL><DEDENT>try:<EOL><INDENT>return expec[:, alt]<EOL><DEDENT>except NotImplementedError:<EOL><INDENT>alt = asarray(alt, int)<EOL>return asarray(expec, float)[:, alt]<EOL><DEDENT>

r""" Compute dosage from allele expectation.

    Parameters
    ----------
    expec : array_like
        Allele expectations encoded as a samples-by-alleles matrix.
    alt : array_like, optional
        Alternative allele index. If ``None``, the allele having the minor
        allele frequency for the provided ``expec`` is used as the alternative.
        Defaults to ``None``.

    Returns
    -------
    :class:`numpy.ndarray`
        Dosage encoded as an array of size equal to the number of samples.

    Examples
    --------
    .. code-block:: python
        :caption: First a quick-start example.

        >>> from bgen_reader import allele_expectation, compute_dosage
        >>> from bgen_reader import example_files, read_bgen
        >>>
        >>> # Download an example.
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> # Read the example.
        >>> bgen = read_bgen(filepath, verbose=False)
        >>>
        >>> # Extract the allele expectations of the fourth variant.
        >>> variant_idx = 3
        >>> e = allele_expectation(bgen, variant_idx)
        >>>
        >>> # Compute the dosage when considering the first allele
        >>> # as the reference/alternative one.
        >>> alt_allele_idx = 1
        >>> d = compute_dosage(e, alt=alt_allele_idx)
        >>>
        >>> # Print the dosage of the first five samples only.
        >>> print(d[:5])
        [1.96185308 0.00982666 0.01745552 1.00347899 1.01153563]
        >>>
        >>> # Clean-up the example
        >>> example.close()

    .. code-block:: python
        :caption: Genotype probabilities, allele expectations and frequencies.

        >>> from bgen_reader import (
        ...     allele_expectation,
        ...     allele_frequency,
        ...     compute_dosage,
        ...     example_files,
        ...     read_bgen,
        ... )
        >>> from pandas import DataFrame
        >>> from xarray import DataArray
        >>>
        >>> # Download an example
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> # Open the bgen file.
        >>> bgen = read_bgen(filepath, verbose=False)
        >>> variants = bgen["variants"]
        >>> genotype = bgen["genotype"]
        >>> samples = bgen["samples"]
        >>>
        >>> variant_idx = 3
        >>> variant = variants.loc[variant_idx].compute()
        >>> # Print the metadata of the fourth variant.
        >>> print(variant)
                id    rsid chrom   pos  nalleles allele_ids  vaddr
        3  SNPID_5  RSID_5    01  5000         2        A,G  16034

        >>> geno = bgen["genotype"][variant_idx].compute()
        >>> metageno = DataFrame({k: geno[k] for k in ["ploidy", "missing"]},
        ...                      index=samples)
        >>> metageno.index.name = "sample"
        >>> print(metageno) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
                    ploidy  missing
        sample
        sample_001       2    False
        sample_002       2    False
        sample_003       2    False
        sample_004       2    False
        ...            ...      ...
        sample_497       2    False
        sample_498       2    False
        sample_499       2    False
        sample_500       2    False
        <BLANKLINE>
        [500 rows x 2 columns]
        >>> p = DataArray(
        ...     geno["probs"],
        ...     name="probability",
        ...     coords={"sample": samples},
        ...     dims=["sample", "genotype"],
        ... )
        >>> # Print the genotype probabilities.
        >>> print(p.to_series().unstack(level=-1)) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
        genotype          0        1        2
        sample
        sample_001  0.00488  0.02838  0.96674
        sample_002  0.99045  0.00928  0.00027
        sample_003  0.98932  0.00391  0.00677
        sample_004  0.00662  0.98328  0.01010
        ...             ...      ...      ...
        sample_497  0.00137  0.01312  0.98550
        sample_498  0.00552  0.99423  0.00024
        sample_499  0.01266  0.01154  0.97580
        sample_500  0.00021  0.98431  0.01547
        <BLANKLINE>
        [500 rows x 3 columns]
        >>> alleles = variant["allele_ids"].item().split(",")
        >>> e = DataArray(
        ...     allele_expectation(bgen, variant_idx),
        ...     name="expectation",
        ...     coords={"sample": samples, "allele": alleles},
        ...     dims=["sample", "allele"],
        ... )
        >>> # Print the allele expectations.
        >>> print(e.to_series().unstack(level=-1)) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
        allele            A        G
        sample
        sample_001  0.03815  1.96185
        sample_002  1.99017  0.00983
        sample_003  1.98254  0.01746
        sample_004  0.99652  1.00348
        ...             ...      ...
        sample_497  0.01587  1.98413
        sample_498  1.00528  0.99472
        sample_499  0.03687  1.96313
        sample_500  0.98474  1.01526
        <BLANKLINE>
        [500 rows x 2 columns]
        >>> rsid = variant["rsid"].item()
        >>> chrom = variant["chrom"].item()
        >>> variant_name = f"{chrom}:{rsid}"
        >>> f = DataFrame(allele_frequency(e), columns=[variant_name], index=alleles)
        >>> f.index.name = "allele"
        >>> # Allele frequencies.
        >>> print(f) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
                01:RSID_5
        allele
        A       305.97218
        G       194.02782
        >>> alt = f.idxmin().item()
        >>> alt_idx = alleles.index(alt)
        >>> d = compute_dosage(e, alt=alt_idx).to_series()
        >>> d = DataFrame(d.values, columns=[f"alt={alt}"], index=d.index)
        >>> # Dosages when considering G as the alternative allele.
        >>> print(d) # doctest: +IGNORE_EXCEPTION_DETAIL, +NORMALIZE_WHITESPACE
                      alt=G
        sample
        sample_001  1.96185
        sample_002  0.00983
        sample_003  0.01746
        sample_004  1.00348
        ...             ...
        sample_497  1.98413
        sample_498  0.99472
        sample_499  1.96313
        sample_500  1.01526
        <BLANKLINE>
        [500 rows x 1 columns]
        >>>
        >>> # Clean-up the example
        >>> example.close()


geno = bgen["<STR_LIT>"][variant_idx].compute()<EOL>if geno["<STR_LIT>"]:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>nalleles = bgen["<STR_LIT>"].loc[variant_idx, "<STR_LIT>"].compute().item()<EOL>genotypes = get_genotypes(geno["<STR_LIT>"], nalleles)<EOL>expec = []<EOL>for i in range(len(genotypes)):<EOL><INDENT>count = asarray(genotypes_to_allele_counts(genotypes[i]), float)<EOL>n = count.shape[<NUM_LIT:0>]<EOL>expec.append((count.T * geno["<STR_LIT>"][i, :n]).sum(<NUM_LIT:1>))<EOL><DEDENT>return stack(expec, axis=<NUM_LIT:0>)<EOL>

r""" Allele expectation.

    Compute the expectation of each allele from the genotype probabilities.

    Parameters
    ----------
    bgen : bgen_file
        Bgen file handler.
    variant_idx : int
        Variant index.

    Returns
    -------
    :class:`numpy.ndarray`
        Samples-by-alleles matrix of allele expectations.

    Note
    ----
    This function supports unphased genotypes only.

    Examples
    --------
    .. doctest::

        >>> from bgen_reader import allele_expectation, example_files, read_bgen
        >>>
        >>> from texttable import Texttable
        >>>
        >>> # Download an example.
        >>> example = example_files("example.32bits.bgen")
        >>> filepath = example.filepath
        >>>
        >>> # Read the example.
        >>> bgen = read_bgen(filepath, verbose=False)
        >>>
        >>> variants = bgen["variants"]
        >>> samples = bgen["samples"]
        >>> genotype = bgen["genotype"]
        >>>
        >>> genotype = bgen["genotype"]
        >>> # This `compute` call will return a pandas data frame,
        >>> variant = variants[variants["rsid"] == "RSID_6"].compute()
        >>> # from which we retrieve the variant index.
        >>> variant_idx = variant.index.item()
        >>> print(variant)
                id    rsid chrom   pos  nalleles allele_ids  vaddr
        4  SNPID_6  RSID_6    01  6000         2        A,G  19377
        >>> genotype = bgen["genotype"]
        >>> # Samples is a pandas series, and we retrieve the
        >>> # sample index from the sample name.
        >>> sample_idx = samples[samples == "sample_005"].index.item()
        >>>
        >>> genotype = bgen["genotype"]
        >>> # This `compute` call will return a dictionary from which
        >>> # we can get the probability matrix the corresponding
        >>> # variant.
        >>> p = genotype[variant_idx].compute()["probs"][sample_idx]
        >>>
        >>> genotype = bgen["genotype"]
        >>> # Allele expectation makes sense for unphased genotypes only,
        >>> # which is the case here.
        >>> e = allele_expectation(bgen, variant_idx)[sample_idx]
        >>>
        >>> genotype = bgen["genotype"]
        >>> alleles = variant["allele_ids"].item().split(",")
        >>>
        >>> genotype = bgen["genotype"]
        >>>
        >>> # Print what we have got in a nice format.
        >>> table = Texttable()
        >>> table = table.add_rows(
        ...     [
        ...         ["", "AA", "AG", "GG", "E[.]"],
        ...         ["p"] + list(p) + ["na"],
        ...         ["#" + alleles[0], 2, 1, 0, e[0]],
        ...         ["#" + alleles[1], 0, 1, 2, e[1]],
        ...     ]
        ... )
        >>> print(table.draw())
        +----+-------+-------+-------+-------+
        |    |  AA   |  AG   |  GG   | E[.]  |
        +====+=======+=======+=======+=======+
        | p  | 0.012 | 0.987 | 0.001 | na    |
        +----+-------+-------+-------+-------+
        | #A | 2     | 1     | 0     | 1.011 |
        +----+-------+-------+-------+-------+
        | #G | 0     | 1     | 2     | 0.989 |
        +----+-------+-------+-------+-------+
        >>>
        >>> # Clean-up.
        >>> example.close()


def read_bgen(filepath, metafile_filepath=None, samples_filepath=None, verbose=True):

assert_file_exist(filepath)<EOL>assert_file_readable(filepath)<EOL>metafile_filepath = _get_valid_metafile_filepath(filepath, metafile_filepath)<EOL>if not os.path.exists(metafile_filepath):<EOL><INDENT>if verbose:<EOL><INDENT>print(<EOL>f"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>create_metafile(filepath, metafile_filepath, verbose)<EOL><DEDENT>samples = get_samples(filepath, samples_filepath, verbose)<EOL>variants = map_metadata(filepath, metafile_filepath)<EOL>genotype = map_genotype(filepath, metafile_filepath, verbose)<EOL>return dict(variants=variants, samples=samples, genotype=genotype)<EOL>

r""" Read a given BGEN file.

    Parameters
    ----------
    filepath : str
        A bgen file path.
    metafile_filepath : str, optional
        If ``None``, it will try to read the ``filepath + ".metadata"`` file. If this is
        not possible, it will create one. It tries to create one at
        ``filepath + ".metadata"``. If that is also no possible, it tries to create one
        at a temporary folder.
    samples_filepath : str, optional
        A sample file in `gen format <https://goo.gl/bCzo7m>`_.
        If ``samples_filepath`` is provided, sample ids are read from this file.
        Otherwise, it reads from the bgen file itself if possible. Defaults to ``None``.
    verbose : bool, optional
        ``True`` to show progress; ``False`` otherwise. Defaults to ``True``.

    Returns
    -------
    variants : :class:`dask.dataFrame.DataFrame`
        Variant position, chromosomes, rsids, etc.
    samples : :class:`pandas.Series`
        Sample identifications.
    genotype : list
        List of genotypes.

    Examples
    --------
    .. doctest::

        >>> from bgen_reader import example_files, read_bgen
        >>>
        >>> with example_files("haplotypes.bgen") as filepath:
        ...     bgen = read_bgen(filepath, verbose=False)
        ...     variants = bgen["variants"]
        ...     samples = bgen["samples"]
        ...
        ...     v = variants.loc[0].compute()
        ...     g = bgen["genotype"][0].compute()
        ...     print(v)
        ...     print(samples)
        ...     print(g["probs"][0])
             id rsid chrom  pos  nalleles allele_ids  vaddr
        0  SNP1  RS1     1    1         2        A,G    102
        0    sample_0
        1    sample_1
        2    sample_2
        3    sample_3
        Name: id, dtype: object
        [1. 0. 1. 0.]


def _touch(fname, mode=<NUM_LIT>, dir_fd=None, **kwargs):

flags = os.O_CREAT | os.O_APPEND<EOL>with os.fdopen(os.open(fname, flags=flags, mode=mode, dir_fd=dir_fd)) as f:<EOL><INDENT>os.utime(<EOL>f.fileno() if os.utime in os.supports_fd else fname,<EOL>dir_fd=None if os.supports_fd else dir_fd,<EOL>**kwargs,<EOL>)<EOL><DEDENT>

Touch a file.

    Credits to <https://stackoverflow.com/a/1160227>.


def create_metafile(bgen_filepath, metafile_filepath, verbose=True):

if verbose:<EOL><INDENT>verbose = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>verbose = <NUM_LIT:0><EOL><DEDENT>bgen_filepath = make_sure_bytes(bgen_filepath)<EOL>metafile_filepath = make_sure_bytes(metafile_filepath)<EOL>assert_file_exist(bgen_filepath)<EOL>assert_file_readable(bgen_filepath)<EOL>if exists(metafile_filepath):<EOL><INDENT>raise ValueError(f"<STR_LIT>")<EOL><DEDENT>with bgen_file(bgen_filepath) as bgen:<EOL><INDENT>nparts = _estimate_best_npartitions(lib.bgen_nvariants(bgen))<EOL>metafile = lib.bgen_create_metafile(bgen, metafile_filepath, nparts, verbose)<EOL>if metafile == ffi.NULL:<EOL><INDENT>raise RuntimeError(f"<STR_LIT>")<EOL><DEDENT>if lib.bgen_close_metafile(metafile) != <NUM_LIT:0>:<EOL><INDENT>raise RuntimeError(f"<STR_LIT>")<EOL><DEDENT><DEDENT>

r"""Create variants metadata file.

    Variants metadata file helps speed up subsequent reads of the associated
    bgen file.

    Parameters
    ----------
    bgen_filepath : str
        Bgen file path.
    metafile_file : str
        Metafile file path.
    verbose : bool
        ``True`` to show progress; ``False`` otherwise.

    Examples
    --------
    .. doctest::

        >>> import os
        >>> from bgen_reader import create_metafile, example_files
        >>>
        >>> with example_files("example.32bits.bgen") as filepath:
        ...     folder = os.path.dirname(filepath)
        ...     metafile_filepath = os.path.join(folder, filepath + ".metadata")
        ...
        ...     try:
        ...         create_metafile(filepath, metafile_filepath, verbose=False)
        ...     finally:
        ...         if os.path.exists(metafile_filepath):
        ...             os.remove(metafile_filepath)


return struct.calcsize("<STR_LIT:P>") * <NUM_LIT:8><EOL>

Determines the number of bits of the Python process.

        Return ``32`` or ``64``.


names = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL>names = [n.format(name) for n in names]<EOL>dirs = self.get_library_dirs()<EOL>for d in dirs:<EOL><INDENT>for n in names:<EOL><INDENT>if exists(join(d, n)):<EOL><INDENT>return n[:-<NUM_LIT:4>]<EOL><DEDENT><DEDENT><DEDENT>msg = "<STR_LIT>".format(name)<EOL>raise ValueError(msg)<EOL>


return miles * float(<NUM_LIT>)<EOL>

Converts a number of miles to feet.

Args:
    miles: Number of miles we want to convert.

Returns:
    Floating point number as the number of
    feet in the given miles.


def total_seconds(hours, minutes, seconds):

return (hours * <NUM_LIT> + minutes) * <NUM_LIT> + seconds<EOL>

Returns the number of seconds in the given number of hours,
minutes, and seconds.

Args:
    hours:
        Integer, number of hours.

    minutes:
        Integer, number of minutes.

    seconds:
        Integer, number of seconds.

Returns:
    Integer, time in seconds.


return width * <NUM_LIT:2> + height * <NUM_LIT:2><EOL>

Returns the perimeter of a rectangle with the given width and height.

Args:
    width:
        Integer or float, width of the rectangle.

    height:
        Integer or float, height of the rectangle.

Returns:
    Integer or float, perimeter of a rectangle.


return width * height<EOL>

Returns the area of a rectangle with the given width and height.

    Args:
        width:
            Integer or float, width of the rectangle.

        height: Integer or float, height of the rectangle.

    Returns:
        The area of a rectangle as an integer or float.


return radius * <NUM_LIT:2> * math.pi<EOL>

Returns the circumference of a circle.

Args:
    radius: The radius of a circle.

Returns:
    Integer > circumference of a circle.

Requires:
    The math module.


return math.pi * radius ** <NUM_LIT:2><EOL>

Returns the area of a circle.

Args:
    radius: The radius of a circle.

Returns:
    The area of a circle as an integer.

Requires:
    The math module.


def compound_interest(principal, annual_rate, years):

return principal * (<NUM_LIT:1> + <NUM_LIT> * annual_rate) ** years<EOL>

Returns the future value of money invested at an annual
interest rate, compounded annually for a given number of years.

Args:
    principal: The beginning ammount of money invested

    annual_rate: The interest rate paid out

    years: The number of years invested

Returns:
    A basic calculation of compound interest.


def future_value(present_value, annual_rate, periods_per_year, years):

<EOL>rate_per_period = annual_rate / float(periods_per_year)<EOL>periods = periods_per_year * years<EOL>return present_value * (<NUM_LIT:1> + rate_per_period) ** periods<EOL>

Calculates the future value of money invested at an anual interest rate,
x times per year, for a given number of years.

Args:
    present_value: int or float, the current value of the money (principal).

    annual_rate: float 0 to 1 e.g., .5 = 50%), the interest rate paid out.

    periods_per_year: int, the number of times money is invested per year.

    years: int, the number of years invested.

Returns:
    Float, the future value of the money invested with compound interest.


return ((point1[<NUM_LIT:0>] - point2[<NUM_LIT:0>]) ** <NUM_LIT:2> + (point1[<NUM_LIT:1>] - point2[<NUM_LIT:1>]) ** <NUM_LIT:2>) ** <NUM_LIT:0.5><EOL>

Computes the distance beteen two points on a plane.

Args:
    point1: Tuple or list, the x and y coordinate of the first point.

    point2: Tuple or list, the x and y coordinate of the second point.

Returns:
    The distance between the two points as a floating point number.


def triangle_area(point1, point2, point3):

"""<STR_LIT>"""<EOL>a = point_distance(point1, point2)<EOL>b = point_distance(point1, point3)<EOL>c = point_distance(point2, point3)<EOL>"""<STR_LIT>"""<EOL>s = (a + b + c) / <NUM_LIT><EOL>"""<STR_LIT>"""<EOL>return math.sqrt(s * (s - a) * (s - b) * (s - c))<EOL>

Uses Heron's formula to find the area of a triangle
based on the coordinates of three points.

Args:
    point1: list or tuple, the x y coordinate of point one.

    point2: list or tuple, the x y coordinate of point two.

    point3: list or tuple, the x y coordinate of point three.

Returns:
    The area of a triangle as a floating point number.

Requires:
    The math module, point_distance().


if (year % <NUM_LIT>) == <NUM_LIT:0>:<EOL><INDENT>return True<EOL><DEDENT>elif (year % <NUM_LIT:100>) == <NUM_LIT:0>:<EOL><INDENT>return False<EOL><DEDENT>elif (year % <NUM_LIT:4>) == <NUM_LIT:0>:<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Checks to see if a given year is a leap year.

Args:
    Integer, the year to test.

Returns:
    Boolean


return (<NUM_LIT> * number_of_sides * length_of_sides ** <NUM_LIT:2>) / math.tan(<EOL>math.pi / number_of_sides<EOL>)<EOL>

Calculates the area of a regular polygon (with sides of equal length).

Args:
    number_of_sides: Integer, the number of sides of the polygon

    length_of_sides: Integer or floating point number, the length of the sides

Returns:
    The area of a regular polygon as an integer or floating point number

Requires:
    The math module


ordered = sorted(data)<EOL>length = len(ordered)<EOL>if length % <NUM_LIT:2> == <NUM_LIT:0>:<EOL><INDENT>return (<EOL>ordered[math.floor(length / <NUM_LIT:2>) - <NUM_LIT:1>] + ordered[math.floor(length / <NUM_LIT:2>)]<EOL>) / <NUM_LIT><EOL><DEDENT>elif length % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>return ordered[math.floor(length / <NUM_LIT:2>)]<EOL><DEDENT>

Calculates  the median of a list of integers or floating point numbers.

Args:
    data: A list of integers or floating point numbers

Returns:
    Sorts the list numerically and returns the middle number if the list has an odd number
    of items. If the list contains an even number of items the mean of the two middle numbers
    is returned.


if type == '<STR_LIT>':<EOL><INDENT>return Decimal(sum(numbers)) / len(numbers)<EOL><DEDENT>else:<EOL><INDENT>return float(sum(numbers)) / len(numbers)<EOL><DEDENT>

Calculates the average or mean of a list of numbers

Args:
    numbers: a list of integers or floating point numbers.

    numtype: string, 'decimal' or 'float'; the type of number to return.

Returns:
    The average (mean) of the numbers as a floating point number
    or a Decimal object.

Requires:
    The math module


mean = average(numbers)<EOL>variance = <NUM_LIT:0><EOL>for number in numbers:<EOL><INDENT>variance += (mean - number) ** <NUM_LIT:2><EOL><DEDENT>if type == '<STR_LIT>':<EOL><INDENT>return variance / len(numbers)<EOL><DEDENT>else:<EOL><INDENT>return variance / (len(numbers) - <NUM_LIT:1>)<EOL><DEDENT>

Calculates the population or sample variance of a list of numbers.
A large number means the results are all over the place, while a
small number means the results are comparatively close to the average.

Args:
    numbers: a list  of integers or floating point numbers to compare.

    type: string, 'population' or 'sample', the kind of variance to be computed.

Returns:
    The computed population or sample variance.
    Defaults to population variance.

Requires:
    The math module, average()


return variance ** <NUM_LIT:0.5><EOL>

Calculates the standard deviation.

Args:
    variance: The variance of a group of numbers.

Returns:
    The standard deviation as a floating point number.


def get_percentage(a, b, i=False, r=False):

<EOL>if i is False and r is True:<EOL><INDENT>percentage = round(<NUM_LIT> * (float(a) / b), <NUM_LIT:2>)<EOL><DEDENT>elif (i is True and r is True) or (i is True and r is False):<EOL><INDENT>percentage = int(round(<NUM_LIT:100> * (float(a) / b)))<EOL>if r is False:<EOL><INDENT>warnings.warn(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>percentage = <NUM_LIT> * (float(a) / b)<EOL><DEDENT>return percentage<EOL>

Finds the percentage of one number over another.

Args:
    a: The number that is a percent, int or float.

    b: The base number that a is a percent of, int or float.

    i: Optional boolean integer. True if the user wants the result returned as
    a whole number. Assumes False.

    r: Optional boolean round. True if the user wants the result rounded.
    Rounds to the second decimal point on floating point numbers. Assumes False.

Returns:
    The argument a as a percentage of b. Throws a warning if integer is set to True
    and round is set to False.


return (float(point2[<NUM_LIT:1>]) - point1[<NUM_LIT:1>]) / (float(point2[<NUM_LIT:0>]) - point1[<NUM_LIT:0>])<EOL>

Calculate the slope of the line connecting two points on a grid.

Args:
    point1: Tuple or list, the x and y coordinate of the first point.

    point2: Tuple or list, the x and y coordinate of the second point

Returns:
    the slope of a line connecting two points on a grid.


return <NUM_LIT:2> ** height<EOL>

Calculate the number of leaves in a complete binary tree in which each internal
node has exactly two children. A full binary tree is complete if every leaf
in the tree has the same depth. A leaf is a node without children

Args:
    height: integer, the height of the tree. Height is defined by the number
    of edges from the furthest child to the root. An edge is the line segment
    that runs between and connects nodes.


return <NUM_LIT:2> ** (height + <NUM_LIT:1>) - <NUM_LIT:1><EOL>

Calculate the number of internal nodes in a complete binary tree in which each
internal node has exactly two children. A full binary tree is complete if every
leaf in the tree has the same depth. Internal nodes include both leaves and
internal nodes. The root node is also included in this calculation.

Args:
    height: integer, the height of the tree. Height is defined by the number
    of edges from the furthest child to the root. An edge is the line segment
    that runs between and connects nodes.


def take_home_pay(gross_pay, employer_match, taxes_and_fees, numtype='<STR_LIT:float>'):

if numtype == '<STR_LIT>':<EOL><INDENT>return (Decimal(gross_pay) + Decimal(employer_match)) - Decimal(<EOL>sum(taxes_and_fees)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>return (float(gross_pay) + float(employer_match)) - sum(taxes_and_fees)<EOL><DEDENT>

Calculate net take-home pay including employer retirement savings match
using the formula laid out by Mr. Money Mustache:
http://www.mrmoneymustache.com/2015/01/26/calculating-net-worth/

Args:
    gross_pay: float or int, gross monthly pay.

    employer_match: float or int, the 401(k) match from your employer.

    taxes_and_fees: list, taxes and fees that are deducted from your paycheck.

    numtype: string, 'decimal' or 'float'; the type of number to return.

Returns:
    your monthly take-home pay.


def savings_rate(take_home_pay, spending, numtype='<STR_LIT:float>'):

if numtype == '<STR_LIT>':<EOL><INDENT>try:<EOL><INDENT>return (<EOL>(Decimal(take_home_pay) - Decimal(spending)) / (Decimal(take_home_pay))<EOL>) * Decimal(<NUM_LIT>)<EOL><DEDENT>except (InvalidOperation, DivisionByZero):<EOL><INDENT>return Decimal(<NUM_LIT:0.0>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>return (<EOL>(float(take_home_pay) - float(spending)) / (float(take_home_pay))<EOL>) * <NUM_LIT><EOL><DEDENT>except (ZeroDivisionError):<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT><DEDENT>

Calculate net take-home pay including employer retirement savings match
using the formula laid out by Mr. Money Mustache:
http://www.mrmoneymustache.com/2015/01/26/calculating-net-worth/

Args:
    take_home_pay: float or int, monthly take-home pay

    spending: float or int, monthly spending

    numtype: string, 'decimal' or 'float'; the type of number to return.

Returns:
    your monthly savings rate expressed as a percentage.


assert a < <NUM_LIT:0><EOL>


self.data_structure = '<STR_LIT>'<EOL>self.built_in = set(['<STR_LIT>', '<STR_LIT>'])<EOL>


return self.data_structure<EOL>


return language in self.built_in<EOL>

Tests to see if a language is a built in type
supported by python.

Args:
    language: string, language to test for.

Returns:
    boolean


def get_built_in(self, language, level, data):

<EOL>pp = pprint.PrettyPrinter(indent=level)<EOL>lookup = {'<STR_LIT>' : pp.pformat(data),<EOL>'<STR_LIT>' : str(json.dumps(data, sort_keys=True, indent=level, separators=('<STR_LIT:U+002C>', '<STR_LIT>')))}<EOL>self.data_structure = lookup[language]<EOL>

Gets the return string for a language that's supported by python.
Used in cases when python provides support for the conversion.

Args:
    language: string the langage to return for.

    level: integer, the indentation level.

    data: python data structure being converted (list of tuples)

Returns:
    None, updates self.data_structure


def get_inner_template(self, language, template_type, indentation, key, val):

<EOL>inner_templates = {'<STR_LIT>' : {<EOL>'<STR_LIT>' : '<STR_LIT>' % (indentation, key, indentation, val, indentation),<EOL>'<STR_LIT>' : '<STR_LIT>' % (indentation, key, val) },<EOL>'<STR_LIT>' : {<EOL>'<STR_LIT>' : '<STR_LIT>' % (indentation, key, val, indentation),<EOL>'<STR_LIT>' : '<STR_LIT>' % (indentation, key, val)},<EOL>'<STR_LIT>' : { <EOL>'<STR_LIT>' : '<STR_LIT>' % (indentation, key, val, indentation),<EOL>'<STR_LIT>' : '<STR_LIT>' % (indentation, key, val)}}<EOL>return inner_templates[language][template_type]<EOL>

Gets the requested template for the given language.

Args:
    language: string, the language of the template to look for.

    template_type: string, 'iterable' or 'singular'.
    An iterable template is needed when the value is an iterable
    and needs more unpacking, e.g. list, tuple. A singular template
    is needed when unpacking is complete and the value is singular,
    e.g. string, int, float.

    indentation: int, the indentation level.

    key: multiple types, the array key.

    val: multiple types, the array values

Returns:
    string, template formatting for arrays by language.


def translate_val(self, language, value):

return self.lang_specific_values[language][value]<EOL>

Translates string representations of language specific
values that vary between languages. Used to translate
python values to their counterparts in other languages.

Args:
    language: string, the language for which to
    return values.

    value: string, the value to translate.

Returns:
    string representation of a value in a given language.


try:<EOL><INDENT>iterate = iter(data)<EOL>return True<EOL><DEDENT>except:<EOL><INDENT>return False<EOL><DEDENT>

Checks to see if an object is an iterable.

Args:
    data: a data object.

Returns:
    boolean


def translate_array(self, string, language, level=<NUM_LIT:3>, retdata=False):

language = language.lower()<EOL>assert self.is_built_in(language) or language in self.outer_templates,"<STR_LIT>" + language + "<STR_LIT>"<EOL>data = phpserialize.loads(bytes(string, '<STR_LIT:utf-8>'), array_hook=list, decode_strings=True)<EOL>if self.is_built_in(language):<EOL><INDENT>self.get_built_in(language, level, data) <EOL>print(self)<EOL>return self.data_structure if retdata else None<EOL><DEDENT>def loop_print(iterable, level=<NUM_LIT:3>):<EOL><INDENT>"""<STR_LIT>"""<EOL>retval = '<STR_LIT>'<EOL>indentation = '<STR_LIT:U+0020>' * level<EOL>if not self.is_iterable(iterable) or isinstance(iterable, str):<EOL><INDENT>non_iterable = str(iterable)<EOL>return str(non_iterable)<EOL><DEDENT>for item in iterable:<EOL><INDENT>if isinstance(item, tuple) and len(item) == <NUM_LIT:2>:<EOL><INDENT>key = item[<NUM_LIT:0>]<EOL>val = loop_print(item[<NUM_LIT:1>], level=level+<NUM_LIT:3>)<EOL>val = self.translate_val(language, val) if language in self.lang_specific_valuesand val in self.lang_specific_values[language] else val<EOL>key = str(key) if isinstance(key, int) else '<STR_LIT>' + str(key) + '<STR_LIT>'<EOL>needs_unpacking = hasattr(item[<NUM_LIT:0>],'<STR_LIT>') == Falseand hasattr(item[<NUM_LIT:1>],'<STR_LIT>') == True <EOL>if needs_unpacking:<EOL><INDENT>retval += self.get_inner_template(language, '<STR_LIT>', indentation, key, val)<EOL><DEDENT>else:<EOL><INDENT>val = str(val) if val.isdigit() or val in self.lang_specific_values[language].values() else '<STR_LIT>' + str(val) + '<STR_LIT>'<EOL>retval += self.get_inner_template(language, '<STR_LIT>', indentation, key, val) <EOL><DEDENT><DEDENT><DEDENT>return retval<EOL><DEDENT>self.data_structure = self.outer_templates[language] % (loop_print(data))<EOL>print(self)<EOL>return self.data_structure if retdata else None<EOL>

Unserializes a serialized php array and prints it to
        the console as a data structure in the specified language.
        Used to translate or convert a php array into a data structure
        in another language. Currently supports, PHP, Python, Javascript,
        and JSON.

        Args:
            string: a string of serialized php

            language: a string representing the desired output
            format for the array.

            level: integer, indentation level in spaces.
            Defaults to 3.

            retdata: boolean, the method will return the string
            in addition to printing it if set to True. Defaults
            to false.

        Returns:
            None but prints a string to the console if retdata is
            False, otherwise returns a string.


description = run_get_percentage.__doc__<EOL>parser = argparse.ArgumentParser(<EOL>prog='<STR_LIT>',<EOL>description=description,<EOL>epilog="<STR_LIT>",<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT:a>', help='<STR_LIT>'<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT:b>',<EOL>help='<STR_LIT>',<EOL>)<EOL>args = parser.parse_args()<EOL>print(sm.get_percentage(float(args.a), float(args.b)))<EOL>

Calculate what percentage a given number is of another,
e.g. 50 is 50% of 100.


<EOL>parser = argparse.ArgumentParser(prog='<STR_LIT>')<EOL>parser.add_argument('<STR_LIT>', nargs='<STR_LIT:?>', help='<STR_LIT>')<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>nargs='<STR_LIT:?>',<EOL>help='<STR_LIT>',<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT>', nargs='<STR_LIT:?>', help='<STR_LIT>'<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT>', action='<STR_LIT:store_true>', help='<STR_LIT>'<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT:-c>', nargs='<STR_LIT:?>', help='<STR_LIT>'<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT>',<EOL>nargs='<STR_LIT:?>',<EOL>help='<STR_LIT>',<EOL>)<EOL>parser.add_argument(<EOL>'<STR_LIT>', action='<STR_LIT:store_true>', help='<STR_LIT>'<EOL>)<EOL>args = parser.parse_args()<EOL>inputs = {<EOL>'<STR_LIT:p>': args.p,<EOL>'<STR_LIT:s>': args.s,<EOL>'<STR_LIT>': args.css,<EOL>'<STR_LIT:m>': args.m,<EOL>'<STR_LIT:c>': args.c,<EOL>'<STR_LIT:d>': args.d,<EOL>'<STR_LIT:r>': args.r,<EOL>}<EOL>p = inputs['<STR_LIT:p>']<EOL>s = inputs['<STR_LIT:s>'] if inputs['<STR_LIT:s>'] else '<STR_LIT>'<EOL>css = inputs['<STR_LIT>'] if inputs['<STR_LIT>'] else '<STR_LIT>'<EOL>m = inputs['<STR_LIT:m>'] if inputs['<STR_LIT:m>'] else False<EOL>c = inputs['<STR_LIT:c>'] if inputs['<STR_LIT:c>'] else '<STR_LIT>'<EOL>d = inputs['<STR_LIT:d>'].split('<STR_LIT:|>') if inputs['<STR_LIT:d>'] else []<EOL>r = inputs['<STR_LIT:r>'] if inputs['<STR_LIT:r>'] else False<EOL>html = fp.excel_to_html(<EOL>p, sheetname=s, css_classes=css, caption=c, details=d, row_headers=r, merge=m<EOL>)<EOL>print(html)<EOL>

Run the excel_to_html function from the
command-line.

Args:
    -p path to file
    -s name of the sheet to convert
    -css classes to apply
    -m attempt to combine merged cells
    -c caption for accessibility
    -su summary for accessibility
    -d details for accessibility

Example use:

    excel_to_html -p myfile.xlsx -s SheetName -css diablo-python -m true


response = urllib.request.urlopen(path)<EOL>return response.read()<EOL>

Gets a file over http.

    Args:
        path: string url of the desired file.

    Returns:
        The desired file as a string.


response = urllib.request.urlopen(file_path)<EOL>f = open(target_path, '<STR_LIT:w>')<EOL>f.write(response.read()) <EOL>f.close()<EOL>

Copies a file from its location on the web to a designated
    place on the local machine.

    Args:
        file_path: Complete url of the file to copy, string (e.g. http://fool.com/input.css).

        target_path: Path and name of file on the local machine, string. (e.g. /directory/output.css)

    Returns:
        None.


i = <NUM_LIT:0><EOL>with open(fname) as f:<EOL><INDENT>for i, l in enumerate(f):<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return i + <NUM_LIT:1><EOL>

Counts the number of lines in a file.

    Args:
        fname: string, name of the file.

    Returns:
        integer, the number of lines in the file.


line_count = get_line_count(f)<EOL>f = open(f, '<STR_LIT>')<EOL>output = open(output, '<STR_LIT>')<EOL>for line in range(line_count):<EOL><INDENT>string = f.readline().rstrip()<EOL>if len(string) > <NUM_LIT:0>:<EOL><INDENT>if string[-<NUM_LIT:1>] == "<STR_LIT:;>":<EOL><INDENT>output.write("<STR_LIT:U+0020>" + string + "<STR_LIT:\n>")<EOL><DEDENT>else:<EOL><INDENT>output.write(string + "<STR_LIT:\n>")<EOL><DEDENT><DEDENT><DEDENT>output.close()<EOL>f.close()<EOL>

Indentes css that has not been indented and saves it to a new file.
    A new file is created if the output destination does not already exist.

    Args:
        f: string, path to file.

        output: string, path/name of the output file (e.g. /directory/output.css).
    print type(response.read())

    Returns:
        None.


line_count = get_line_count(f)<EOL>f = open(f, '<STR_LIT>')<EOL>output = open(output, '<STR_LIT>')<EOL>for line in range(line_count):<EOL><INDENT>string = f.readline()<EOL>string = re.sub(char, char + '<STR_LIT:\n>', string)<EOL>output.write(string)<EOL><DEDENT>

Adds line breaks after every occurance of a given character in a file.

    Args:
        f: string, path to input file.

        output: string, path to output file.

    Returns:
        None.


def add_whitespace_before(char, input_file, output_file):

line_count = get_line_count(input_file)<EOL>input_file = open(input_file, '<STR_LIT:r>')<EOL>output_file = open(output_file, '<STR_LIT>')<EOL>for line in range(line_count):<EOL><INDENT>string = input_file.readline()<EOL>if re.search(r'<STR_LIT>' + char, string) != None:<EOL><INDENT>string = re.sub(char, '<STR_LIT:U+0020>' + char, string)<EOL><DEDENT>output_file.write(string)<EOL><DEDENT>input_file.close()<EOL>

Adds a space before a character if there's isn't one already.

    Args:
        char: string, character that needs a space before it.

        input_file: string, path to file to parse.

        output_file: string, path to destination file.

    Returns:
        None.


<EOL>line_count = get_line_count(input_file)<EOL>f = open(input_file, '<STR_LIT>')<EOL>output = open(output_file, '<STR_LIT:w>')<EOL>for line in range(line_count):<EOL><INDENT>string = f.readline().strip()<EOL>string = re.sub('<STR_LIT>', '<STR_LIT>', string)<EOL>string = re.sub('<STR_LIT>', '<STR_LIT:;>', string)<EOL>string = re.sub('<STR_LIT:;>', '<STR_LIT>', string)<EOL>string = re.sub('<STR_LIT>', '<STR_LIT>', string)<EOL>string = re.sub('<STR_LIT>', '<STR_LIT>', string)<EOL>string = re.sub('<STR_LIT>', '<STR_LIT>', string)<EOL>output.write(string)<EOL><DEDENT>output.close()<EOL>f.close()<EOL>indent_css(output_file, output_file)<EOL>add_whitespace_before("<STR_LIT:{>", output_file, output_file)<EOL>

Reformats poorly written css. This function does not validate or fix errors in the code.
    It only gives code the proper indentation.

    Args:
        input_file: string, path to the input file.

        output_file: string, path to where the reformatted css should be saved. If the target file
        doesn't exist, a new file is created.

    Returns:
        None.


try:<EOL><INDENT>float(string)<EOL>return True<EOL><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>

Checks if a string is numeric. If the string value is an integer
or a float, return True, otherwise False. Can be used to test
soley for floats as well.

Args:
    string: a string to test.

Returns:
    boolean


if is_numeric(num):<EOL><INDENT>try:<EOL><INDENT>num / <NUM_LIT:2><EOL>return True<EOL><DEDENT>except:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return False<EOL>

Test to see if an argument is an acutal number.
Returns True if passed an int, float, or decimal,
otherwise False.

Args:
    num: int, float, decimal, or string.

Returns:
    boolean


for string in string_list:<EOL><INDENT>if not is_numeric(string):<EOL><INDENT>return string<EOL><DEDENT><DEDENT>return True<EOL>

Checks a list of strings to see that all values in the list are
numeric. Returns the name of the offending string if it is
not numeric.

Args:
    string_list: a list of strings to test.

Returns:
    boolean or string


try:<EOL><INDENT>a = float(string)<EOL>b = int(a)<EOL><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return a == b<EOL><DEDENT>

Checks if a string is an integer. If the string value is an integer
return True, otherwise return False.

Args:
    string: a string to test.

Returns:
    boolean


hours = <NUM_LIT:0> <EOL>allow = set(['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>for input_file in input_files:<EOL><INDENT>doc = open(input_file, '<STR_LIT:r>')<EOL>for line in doc:<EOL><INDENT>line = line.rstrip()<EOL>data = line.split('<STR_LIT:U+0020>')<EOL>if (len(data) == <NUM_LIT:2>) and (is_numeric(data[<NUM_LIT:0>])) and (data[<NUM_LIT:1>].lower() in allow):<EOL><INDENT>hours += float(data[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>doc.close()<EOL><DEDENT>return hours<EOL>

Totals the hours for a given projct. Takes a list of input files for
which to total the hours. Each input file represents a project.
There are only multiple files for the same project when the duration
was more than a year. A typical entry in an input file might look
like this:

8/24/14
9:30-12:00 wrote foobar code for x, wrote a unit test for foobar code, tested.
2.5 hours

Args:
    input_files: a list of files to parse.

Returns:
    float: the total number of hours spent on the project.


retval = []<EOL>for val in iterable:<EOL><INDENT>try:<EOL><INDENT>retval.append(val.strip())<EOL><DEDENT>except(AttributeError):<EOL><INDENT>retval.append(val)<EOL><DEDENT><DEDENT>return retval<EOL>

Take a list of strings and clear whitespace
on each one. If a value in the list is not a
string pass it through untouched.

Args:
    iterable: mixed list

Returns:
    mixed list


try:<EOL><INDENT>json_object = json.loads(data)<EOL><DEDENT>except(ValueError):<EOL><INDENT>return False<EOL><DEDENT>return True<EOL>

Test if a string is valid json.

Args:
    data: string

Returns:
    boolean


def excel_to_html(path, sheetname='<STR_LIT>', css_classes='<STR_LIT>',caption='<STR_LIT>', details=[], row_headers=False, merge=False):

def get_data_on_merged_cells():<EOL><INDENT>"""<STR_LIT>"""<EOL>merged_cells = xls.book.sheet_by_name(sheetname).merged_cells<EOL>ds = {}<EOL>for crange in merged_cells:<EOL><INDENT>rlo, rhi, clo, chi = crange<EOL>for rowx in range(rlo, rhi):<EOL><INDENT>for colx in range(clo, chi):<EOL><INDENT>parent_cell = (rlo,clo)<EOL>child_cell = (rowx,colx)<EOL>if not parent_cell in ds:<EOL><INDENT>ds[parent_cell] = [[<NUM_LIT:1>,<NUM_LIT:1>], set([])]<EOL><DEDENT>else:<EOL><INDENT>if parent_cell != child_cell and child_cell[<NUM_LIT:0>] == parent_cell[<NUM_LIT:0>]:<EOL><INDENT>ds[parent_cell][<NUM_LIT:0>][<NUM_LIT:0>] += <NUM_LIT:1><EOL>ds[parent_cell][<NUM_LIT:1>].add('<STR_LIT:right>')<EOL><DEDENT>elif parent_cell != child_cell and child_cell[<NUM_LIT:0>] > parent_cell[<NUM_LIT:0>]:<EOL><INDENT>if child_cell[<NUM_LIT:1>] == parent_cell[<NUM_LIT:1>]:<EOL><INDENT>ds[parent_cell][<NUM_LIT:0>][<NUM_LIT:1>] += <NUM_LIT:1><EOL><DEDENT>ds[parent_cell][<NUM_LIT:1>].add('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return ds<EOL><DEDENT>def mark_cells_going_right(cell, curr_cell, merged_cells):<EOL><INDENT>"""<STR_LIT>"""<EOL>try:<EOL><INDENT>xcount = merged_cells[curr_cell][<NUM_LIT:0>][<NUM_LIT:0>]<EOL>if xcount > <NUM_LIT:1>: <EOL><INDENT>cell['<STR_LIT>'] = xcount<EOL><DEDENT>col_count = xcount - <NUM_LIT:1> <EOL>while col_count > <NUM_LIT:0>:<EOL><INDENT>cell = cell.find_next_sibling()<EOL>cell['<STR_LIT:class>'] = '<STR_LIT>'<EOL>col_count -= <NUM_LIT:1><EOL><DEDENT><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>def mark_cells_going_down(cell, curr_cell, merged_cells):<EOL><INDENT>"""<STR_LIT>"""<EOL>if curr_cell in merged_cells and merged_cells[curr_cell][<NUM_LIT:1>] == set(['<STR_LIT>']):<EOL><INDENT>ycount = merged_cells[curr_cell][<NUM_LIT:0>][<NUM_LIT:1>]<EOL>cell['<STR_LIT>'] = ycount<EOL>row_count = ycount<EOL>for child_row in cell.parent.find_next_siblings(limit=row_count - <NUM_LIT:1>):<EOL><INDENT>i = <NUM_LIT:0><EOL>for child in child_row.find_all('<STR_LIT>'):<EOL><INDENT>if i == curr_cell[<NUM_LIT:1>]:<EOL><INDENT>child['<STR_LIT:class>'] = '<STR_LIT>'<EOL><DEDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>def mark_cells_going_down_and_right(cell, curr_cell, merged_cells):<EOL><INDENT>"""<STR_LIT>"""<EOL>if curr_cell in merged_cells and('<STR_LIT>' in merged_cells[curr_cell][<NUM_LIT:1>] and'<STR_LIT:right>' in merged_cells[curr_cell][<NUM_LIT:1>]):<EOL><INDENT>xcount = merged_cells[curr_cell][<NUM_LIT:0>][<NUM_LIT:0>]<EOL>ycount = merged_cells[curr_cell][<NUM_LIT:0>][<NUM_LIT:1>]<EOL>row_count = ycount<EOL>col_count = xcount<EOL>mark_cells_going_right(cell, curr_cell, merged_cells)<EOL>flag = False<EOL>for child_row in [cell.parent] + cell.parent.find_all_next('<STR_LIT>', limit=row_count - <NUM_LIT:1>):<EOL><INDENT>i = <NUM_LIT:0><EOL>for child in child_row.find_all('<STR_LIT>'):<EOL><INDENT>if i == curr_cell[<NUM_LIT:1>]:<EOL><INDENT>mark_cells_going_right(child, curr_cell, merged_cells)<EOL>if not flag:<EOL><INDENT>child['<STR_LIT>'] = col_count<EOL>child['<STR_LIT>'] = row_count<EOL>flag = True<EOL><DEDENT>else:<EOL><INDENT>child['<STR_LIT:class>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>i += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>def is_empty_th(string):<EOL><INDENT>"""<STR_LIT>"""<EOL>if string[:<NUM_LIT:8>] == '<STR_LIT>':<EOL><INDENT>data = string.split('<STR_LIT:U+0020>')<EOL>if is_numeric(data[<NUM_LIT:1>]):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL><DEDENT>def mark_header_cells(html):<EOL><INDENT>"""<STR_LIT>"""<EOL>th = html.find_all('<STR_LIT>')<EOL>for header in th:<EOL><INDENT>txt = header.string<EOL>if not is_empty_th(txt):<EOL><INDENT>header['<STR_LIT>'] = '<STR_LIT>'<EOL>count = <NUM_LIT:1><EOL>for sibling in header.find_next_siblings():<EOL><INDENT>if is_empty_th(sibling.string):<EOL><INDENT>count += <NUM_LIT:1><EOL>sibling['<STR_LIT:class>'] = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if count > <NUM_LIT:1>:<EOL><INDENT>header['<STR_LIT>'] = count<EOL>header['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT><DEDENT><DEDENT>def create_caption(html, caption):<EOL><INDENT>"""<STR_LIT>"""<EOL>ctag = html.new_tag('<STR_LIT>')<EOL>ctag.insert(<NUM_LIT:0>, caption)<EOL>html.table.insert(<NUM_LIT:0>, ctag)<EOL><DEDENT>def create_summary_and_details(html, details):<EOL><INDENT>"""<STR_LIT>"""<EOL>if len(details) != <NUM_LIT:2>:<EOL><INDENT>msg = '<STR_LIT>'+ '<STR_LIT>'+ '<STR_LIT>'+ '<STR_LIT>'<EOL>raise RuntimeError(msg)<EOL><DEDENT>summary = details[<NUM_LIT:0>]<EOL>details = details[<NUM_LIT:1>]<EOL>if not caption:<EOL><INDENT>create_caption(html, caption)<EOL><DEDENT>dtag = html.new_tag('<STR_LIT>')<EOL>stag = html.new_tag('<STR_LIT>')<EOL>ptag = html.new_tag('<STR_LIT:p>')<EOL>stag.insert(<NUM_LIT:0>, summary)<EOL>ptag.insert(<NUM_LIT:0>, details)<EOL>dtag.insert(<NUM_LIT:0>, stag)<EOL>dtag.append(ptag) <EOL>html.table.caption.insert(<NUM_LIT:1>, dtag)   <EOL><DEDENT>def format_properly(html):<EOL><INDENT>"""<STR_LIT>"""<EOL>return html.replace('<STR_LIT>', '<STR_LIT>').replace('<STR_LIT>','<STR_LIT>').replace('<STR_LIT>', '<STR_LIT>').replace('<STR_LIT>','<STR_LIT>').replace('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>def add_row_headers(html):<EOL><INDENT>"""<STR_LIT>"""<EOL>for row in html.tbody.find_all('<STR_LIT>'):<EOL><INDENT>spans_rows = '<STR_LIT>' in row.td.attrs<EOL>spans_columns = '<STR_LIT>' in row.td.attrs<EOL>new_tag = html.new_tag('<STR_LIT>')<EOL>new_tag['<STR_LIT>'] = '<STR_LIT>'<EOL>new_tag.string = row.td.string<EOL>if spans_rows:<EOL><INDENT>new_tag['<STR_LIT>'] = row.td.attrs['<STR_LIT>']<EOL>new_tag['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if spans_columns:<EOL><INDENT>new_tag['<STR_LIT>'] = row.td.attrs['<STR_LIT>']<EOL><DEDENT>row.td.replace_with(new_tag)<EOL><DEDENT><DEDENT>def beautify(html):<EOL><INDENT>"""<STR_LIT>"""<EOL>table = html.find('<STR_LIT>')<EOL>first_tr = table.find('<STR_LIT>')<EOL>del table['<STR_LIT>']<EOL>del first_tr['<STR_LIT>']<EOL>return format_properly(html.prettify(formatter='<STR_LIT>'))<EOL><DEDENT>def parse_html(html, caption, details):<EOL><INDENT>"""<STR_LIT>"""<EOL>new_html = BeautifulSoup(html, '<STR_LIT>')<EOL>if merge:<EOL><INDENT>row_num = <NUM_LIT:1><EOL>merged_cells = get_data_on_merged_cells()<EOL>rows = new_html.find('<STR_LIT>').find('<STR_LIT>').find_all('<STR_LIT>')<EOL>for row in rows:<EOL><INDENT>cell_num = <NUM_LIT:0> <EOL>cells = row.find_all('<STR_LIT>')<EOL>for cell in cells:<EOL><INDENT>curr_cell = (row_num, cell_num)<EOL>mark_cells_going_right(cell, curr_cell, merged_cells)  <EOL>mark_cells_going_down(cell, curr_cell, merged_cells)<EOL>mark_cells_going_down_and_right(cell, curr_cell, merged_cells)<EOL>cell_num += <NUM_LIT:1><EOL><DEDENT>row_num += <NUM_LIT:1><EOL><DEDENT>mark_header_cells(new_html)<EOL>destroy = new_html.find_all(attrs={'<STR_LIT:class>' : '<STR_LIT>' })<EOL>for item in destroy:<EOL><INDENT>item.extract()<EOL><DEDENT><DEDENT>if row_headers:<EOL><INDENT>add_row_headers(new_html)<EOL><DEDENT>if caption:<EOL><INDENT>create_caption(new_html, caption)<EOL><DEDENT>if details:<EOL><INDENT>create_summary_and_details(new_html, details)<EOL><DEDENT>return beautify(new_html)<EOL><DEDENT>pd.options.display.max_colwidth = -<NUM_LIT:1><EOL>xls = pd.ExcelFile(path)<EOL>df = xls.parse(sheetname)<EOL>panda_html = df.to_html(classes=css_classes, index=False, na_rep='<STR_LIT>')<EOL>return parse_html(panda_html, caption, details)<EOL>

Convert an excel spreadsheet to an html table.
This function supports the conversion of merged
cells. It can be used in code or run from the
command-line. If passed the correct arguments
it can generate fully accessible html.

Args:
    path: string, path to the spreadsheet.

    sheetname: string, name of the sheet
    to convert.

    css_classes: string, space separated
    classnames to append to the table.

    caption: string, a short heading-like
    description of the table.

    details: list of strings, where the first
    item in the list is a string for the html
    summary element and the second item is
    a string for the details element. The
    summary should be very short, e.g. "Help",
    where as the details element should be a
    long description regarding the purpose or
    how to navigate the table.

    row_headers: boolean, defaults to False.
    Does the table have row headers? If set
    to True, the first element in each row
    will be a <th scope="row"> element
    instead of a <td> element.

    merge: boolean, whether or not to
    combine cells that were merged in the
    spreadsheet.

Returns:
    string, html table


try:<EOL><INDENT>f(*args)<EOL><DEDENT>except AssertionError:<EOL><INDENT>return True<EOL><DEDENT>except Exception:<EOL><INDENT>raise AssertionError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise AssertionError("<STR_LIT>")<EOL><DEDENT>

Test if an AssertionError is thrown. This
function expects to trigger an Assertion
Error. If an AssertionError is triggered
nothing happens. If a different kind of
exception is triggered, an AssertionError
is raised. If no exception is triggered
an AssertionError is raised.

Args:
    f: function to test.

    *args: unamed arguments.

Returns:
    True or raises an AssertionError.


def camelize_classname(base, tablename, table):

"<STR_LIT>"<EOL>return str(tablename[<NUM_LIT:0>].upper() +<EOL>re.sub(r'<STR_LIT>', lambda m: m.group(<NUM_LIT:1>).upper(),<EOL>tablename[<NUM_LIT:1>:]))<EOL>


def pluralize_collection(base, local_cls, referred_cls, constraint):

"<STR_LIT>"<EOL>referred_name = referred_cls.__name__<EOL>uncamelized = re.sub(r'<STR_LIT>',<EOL>lambda m: "<STR_LIT>" % m.group(<NUM_LIT:0>).lower(),<EOL>referred_name)[<NUM_LIT:1>:]<EOL>pluralized = _pluralizer.plural(uncamelized)<EOL>return pluralized<EOL>

Produce an 'uncamelized', 'pluralized' class name, e.g.


super().__init__(*args, **kwargs)<EOL>self.last_populate_args = []<EOL>self.last_populate_kwargs = {}<EOL>


return self._count_model(Model)<EOL>


return self._list_model(Model)<EOL>


return <NUM_LIT:0> < self.count_model()<EOL>

Check if the database is already populated.


def populate(self, *args, **kwargs) -> None:

models = [<EOL>Model.from_id(model_id)<EOL>for model_id in range(NUMBER_TEST_MODELS)<EOL>]<EOL>self.session.add_all(models)<EOL>self.session.commit()<EOL>if args:<EOL><INDENT>self.last_populate_args = args<EOL>log.critical('<STR_LIT>', args)<EOL><DEDENT>if kwargs:<EOL><INDENT>self.last_populate_kwargs = kwargs<EOL>log.critical('<STR_LIT>', kwargs)<EOL><DEDENT>


def summarize(self) -> Mapping[str, int]:

return dict(<EOL>models=self.count_model(),<EOL>)<EOL>


cls.manager.populate()<EOL>


cls.manager.populate(return_true=True)<EOL>


self.runner = CliRunner()<EOL>self.main = Manager.get_cli()<EOL>

Set up a CliRunner and an accompanying CLI for each test.


self.manager.populate()<EOL>


self.runner = CliRunner()<EOL>self.main = NamespaceManager.get_cli()<EOL>self.manager = Manager(connection=self.connection)<EOL>self.manager.populate()<EOL>


def make_downloader(url: str, path: str) -> Callable[[bool], str]:

def download_data(force_download: bool = False) -> str:<EOL><INDENT>"""<STR_LIT>"""<EOL>if os.path.exists(path) and not force_download:<EOL><INDENT>log.info('<STR_LIT>', path)<EOL><DEDENT>else:<EOL><INDENT>log.info('<STR_LIT>', url, path)<EOL>urlretrieve(url, path)<EOL><DEDENT>return path<EOL><DEDENT>return download_data<EOL>

Make a function that downloads the data for you, or uses a cached version at the given path.

    :param url: The URL of some data
    :param path: The path of the cached data, or where data is cached if it does not already exist
    :return: A function that downloads the data and returns the path of the data


def make_df_getter(data_url: str, data_path: str, **kwargs) -> Callable[[Optional[str], bool, bool], pd.DataFrame]:

download_function = make_downloader(data_url, data_path)<EOL>def get_df(url: Optional[str] = None, cache: bool = True, force_download: bool = False) -> pd.DataFrame:<EOL><INDENT>"""<STR_LIT>"""<EOL>if url is None and cache:<EOL><INDENT>url = download_function(force_download=force_download)<EOL><DEDENT>return pd.read_csv(<EOL>url or data_url,<EOL>**kwargs<EOL>)<EOL><DEDENT>return get_df<EOL>

Build a function that handles downloading tabular data and parsing it into a pandas DataFrame.

    :param data_url: The URL of the data
    :param data_path: The path where the data should get stored
    :param kwargs: Any other arguments to pass to :func:`pandas.read_csv`


def make_zipped_df_getter(data_url: str, data_path: str, zip_path: str, **kwargs):

download_function = make_downloader(data_url, data_path)<EOL>def get_df(url: Optional[str] = None, cache: bool = True, force_download: bool = False) -> pd.DataFrame:<EOL><INDENT>"""<STR_LIT>"""<EOL>if url is not None:<EOL><INDENT>return pd.read_csv(url, **kwargs)<EOL><DEDENT>if url is None and cache:<EOL><INDENT>url = download_function(force_download=force_download)<EOL><DEDENT>with ZipFile(url) as zip_file:<EOL><INDENT>with zip_file.open(zip_path) as file:<EOL><INDENT>return pd.read_csv(file, **kwargs)<EOL><DEDENT><DEDENT><DEDENT>return get_df<EOL>

:param data_url:
:param data_path:
:param zip_path:
:return:


@classmethod<EOL><INDENT>def get_cli(cls) -> click.Group:<DEDENT>

group_help = '<STR_LIT>'.format(cls._get_connection(), get_version())<EOL>@click.group(help=group_help)<EOL>@click.option('<STR_LIT:-c>', '<STR_LIT>', default=cls._get_connection(),<EOL>help='<STR_LIT>'.format(cls._get_connection()))<EOL>@click.pass_context<EOL>def main(ctx, connection):<EOL><INDENT>"""<STR_LIT>"""<EOL>logging.basicConfig(level=logging.INFO, format="<STR_LIT>")<EOL>logging.getLogger('<STR_LIT>').setLevel(logging.WARNING)<EOL>ctx.obj = cls(connection=connection)<EOL><DEDENT>return main<EOL>

Build a :mod:`click` CLI main function.

        :param Type[AbstractManager] cls: A Manager class
        :return: The main function for click


def build_engine_session(connection, echo=False, autoflush=None, autocommit=None, expire_on_commit=None,<EOL>scopefunc=None):

if connection is None:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>engine = create_engine(connection, echo=echo)<EOL>autoflush = autoflush if autoflush is not None else False<EOL>autocommit = autocommit if autocommit is not None else False<EOL>expire_on_commit = expire_on_commit if expire_on_commit is not None else True<EOL>log.debug('<STR_LIT>', autoflush, autocommit, expire_on_commit)<EOL>session_maker = sessionmaker(<EOL>bind=engine,<EOL>autoflush=autoflush,<EOL>autocommit=autocommit,<EOL>expire_on_commit=expire_on_commit,<EOL>)<EOL>session = scoped_session(<EOL>session_maker,<EOL>scopefunc=scopefunc<EOL>)<EOL>return engine, session<EOL>

Build an engine and a session.

    :param str connection: An RFC-1738 database connection string
    :param bool echo: Turn on echoing SQL
    :param Optional[bool] autoflush: Defaults to True if not specified in kwargs or configuration.
    :param Optional[bool] autocommit: Defaults to False if not specified in kwargs or configuration.
    :param Optional[bool] expire_on_commit: Defaults to False if not specified in kwargs or configuration.
    :param scopefunc: Scoped function to pass to :func:`sqlalchemy.orm.scoped_session`
    :rtype: tuple[Engine,Session]

    From the Flask-SQLAlchemy documentation:

    An extra key ``'scopefunc'`` can be set on the ``options`` dict to
    specify a custom scope function.  If it's not provided, Flask's app
    context stack identity is used. This will ensure that sessions are
    created and removed with the request/response cycle, and should be fine
    in most cases.


def __init__(self, connection: Optional[str] = None, engine=None, session=None, **kwargs):

self._assert_module_name()<EOL>if connection and (engine or session):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if engine is None and session is None:<EOL><INDENT>if connection is None:<EOL><INDENT>connection = self._get_connection()<EOL><DEDENT>engine, session = build_engine_session(connection=connection, **kwargs)<EOL><DEDENT>self.engine = engine<EOL>self.session = session<EOL>create_all(self.engine)<EOL>

Build an abstract manager from either a connection or an engine/session.

        The remaining keyword arguments are passed to :func:`build_engine_session`.

        :param Optional[str] connection:
        :param engine:
        :param session:


@property<EOL><INDENT>def connection(self) -> str:<DEDENT>

return str(self.engine.url)<EOL>

Return this manager's connection string.


@classmethod<EOL><INDENT>def _get_connection(cls, connection: Optional[str] = None) -> str:<DEDENT>

return get_connection(cls.module_name, connection=connection)<EOL>

Get a default connection string.

        Wraps :func:`bio2bel.utils.get_connection` and passing this class's :data:`module_name` to it.


def add_cli_to_bel_namespace(main: click.Group) -> click.Group:

@main.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', is_flag=True)<EOL>@click.pass_obj<EOL>def upload(manager: BELNamespaceManagerMixin, update):<EOL><INDENT>"""<STR_LIT>"""<EOL>namespace = manager.upload_bel_namespace(update=update)<EOL>click.echo(f'<STR_LIT>')<EOL><DEDENT>return main<EOL>

Add a ``upload_bel_namespace`` command to main :mod:`click` function.


def add_cli_clear_bel_namespace(main: click.Group) -> click.Group:

@main.command()<EOL>@click.pass_obj<EOL>def drop(manager: BELNamespaceManagerMixin):<EOL><INDENT>"""<STR_LIT>"""<EOL>namespace = manager.drop_bel_namespace()<EOL>if namespace:<EOL><INDENT>click.echo(f'<STR_LIT>')<EOL><DEDENT><DEDENT>return main<EOL>

Add a ``clear_bel_namespace`` command to main :mod:`click` function.


def add_cli_write_bel_namespace(main: click.Group) -> click.Group:

@main.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', type=click.Path(file_okay=False, dir_okay=True), default=os.getcwd(),<EOL>help='<STR_LIT>')<EOL>@click.pass_obj<EOL>def write(manager: BELNamespaceManagerMixin, directory: str):<EOL><INDENT>"""<STR_LIT>"""<EOL>manager.write_directory(directory)<EOL><DEDENT>return main<EOL>

Add a ``write_bel_namespace`` command to main :mod:`click` function.


def add_cli_write_bel_annotation(main: click.Group) -> click.Group:

@main.command()<EOL>@click.option('<STR_LIT>', '<STR_LIT>', type=click.Path(file_okay=False, dir_okay=True), default=os.getcwd(),<EOL>help='<STR_LIT>')<EOL>@click.pass_obj<EOL>def write(manager: BELNamespaceManagerMixin, directory: str):<EOL><INDENT>"""<STR_LIT>"""<EOL>with open(os.path.join(directory, manager.identifiers_namespace), '<STR_LIT:w>') as file:<EOL><INDENT>manager.write_bel_annotation(file)<EOL><DEDENT><DEDENT>return main<EOL>

Add a ``write_bel_annotation`` command to main :mod:`click` function.


@abstractmethod<EOL><INDENT>def _create_namespace_entry_from_model(self, model, namespace: Namespace) -> NamespaceEntry:<DEDENT>

Create a PyBEL NamespaceEntry model from a Bio2BEL model.

        :param model: The model to convert
        :param namespace: The PyBEL namespace to add to


@classmethod<EOL><INDENT>def _get_identifier(cls, model) -> str:<DEDENT>

return getattr(model, f'<STR_LIT>')<EOL>

Extract the identifier from an instance of namespace_model.

        :param model: The model to convert


@staticmethod<EOL><INDENT>def _get_encoding(model) -> str:<DEDENT>

return model.bel_encoding<EOL>

Extract the BEL encoding from an instance of a namespace_model.

        :param model: The model to convert


@staticmethod<EOL><INDENT>def _get_name(model) -> str:<DEDENT>

return model.name<EOL>

Extract the name from an instance of namespace_model.

        :param model: The model to convert


def _iterate_namespace_models(self, **kwargs) -> Iterable:

return tqdm(<EOL>self._get_query(self.namespace_model),<EOL>total=self._count_model(self.namespace_model),<EOL>**kwargs<EOL>)<EOL>

Return an iterator over the models to be converted to the namespace.


@classmethod<EOL><INDENT>def _get_namespace_name(cls) -> str:<DEDENT>

return cls.identifiers_recommended or cls.module_name<EOL>

Get the nicely formatted name of this namespace.


@classmethod<EOL><INDENT>def _get_namespace_keyword(cls) -> str:<DEDENT>

return cls.identifiers_namespace or cls.module_name.upper()<EOL>

Get the keyword to use as the reference BEL namespace.


@classmethod<EOL><INDENT>def _get_namespace_url(cls) -> str:<DEDENT>

return cls.identifiers_url or f'<STR_LIT>'<EOL>

Get the URL to use as the reference BEL namespace.


def _get_default_namespace(self) -> Optional[Namespace]:

return self._get_query(Namespace).filter(Namespace.url == self._get_namespace_url()).one_or_none()<EOL>

Get the reference BEL namespace if it exists.


namespace = Namespace(<EOL>name=self._get_namespace_name(),<EOL>keyword=self._get_namespace_keyword(),<EOL>url=self._get_namespace_url(),<EOL>version=str(time.asctime()),<EOL>)<EOL>self.session.add(namespace)<EOL>entries = self._get_namespace_entries(namespace)<EOL>self.session.add_all(entries)<EOL>t = time.time()<EOL>log.info('<STR_LIT>')<EOL>self.session.commit()<EOL>log.info('<STR_LIT>', time.time() - t)<EOL>return namespace<EOL>


@staticmethod<EOL><INDENT>def _get_old_entry_identifiers(namespace: Namespace) -> Set[NamespaceEntry]:<DEDENT>

return {term.identifier for term in namespace.entries}<EOL>

Convert a PyBEL generalized namespace entries to a set.

        Default to using the identifier, but can be overridden to use the name instead.

        >>> {term.identifier for term in namespace.entries}


def _update_namespace(self, namespace: Namespace) -> None:

old_entry_identifiers = self._get_old_entry_identifiers(namespace)<EOL>new_count = <NUM_LIT:0><EOL>skip_count = <NUM_LIT:0><EOL>for model in self._iterate_namespace_models():<EOL><INDENT>if self._get_identifier(model) in old_entry_identifiers:<EOL><INDENT>continue<EOL><DEDENT>entry = self._create_namespace_entry_from_model(model, namespace=namespace)<EOL>if entry is None or entry.name is None:<EOL><INDENT>skip_count += <NUM_LIT:1><EOL>continue<EOL><DEDENT>new_count += <NUM_LIT:1><EOL>self.session.add(entry)<EOL><DEDENT>t = time.time()<EOL>log.info('<STR_LIT>', new_count, skip_count)<EOL>self.session.commit()<EOL>log.info('<STR_LIT>', time.time() - t)<EOL>

Update an already-created namespace.

        Note: Only call this if namespace won't be none!


def add_namespace_to_graph(self, graph: BELGraph) -> Namespace:

namespace = self.upload_bel_namespace()<EOL>graph.namespace_url[namespace.keyword] = namespace.url<EOL>self._add_annotation_to_graph(graph)<EOL>return namespace<EOL>

Add this manager's namespace to the graph.


def _add_annotation_to_graph(self, graph: BELGraph) -> None:

if '<STR_LIT>' not in graph.annotation_list:<EOL><INDENT>graph.annotation_list['<STR_LIT>'] = set()<EOL><DEDENT>graph.annotation_list['<STR_LIT>'].add(self.module_name)<EOL>

Add this manager as an annotation to the graph.


def upload_bel_namespace(self, update: bool = False) -> Namespace:

if not self.is_populated():<EOL><INDENT>self.populate()<EOL><DEDENT>namespace = self._get_default_namespace()<EOL>if namespace is None:<EOL><INDENT>log.info('<STR_LIT>', self._get_namespace_name())<EOL>return self._make_namespace()<EOL><DEDENT>if update:<EOL><INDENT>self._update_namespace(namespace)<EOL><DEDENT>return namespace<EOL>

Upload the namespace to the PyBEL database.

        :param update: Should the namespace be updated first?


def drop_bel_namespace(self) -> Optional[Namespace]:

namespace = self._get_default_namespace()<EOL>if namespace is not None:<EOL><INDENT>for entry in tqdm(namespace.entries, desc=f'<STR_LIT>'):<EOL><INDENT>self.session.delete(entry)<EOL><DEDENT>self.session.delete(namespace)<EOL>log.info('<STR_LIT>')<EOL>self.session.commit()<EOL>return namespace<EOL><DEDENT>

Remove the default namespace if it exists.


def write_bel_namespace(self, file: TextIO, use_names: bool = False) -> None:

if not self.is_populated():<EOL><INDENT>self.populate()<EOL><DEDENT>if use_names and not self.has_names:<EOL><INDENT>raise ValueError<EOL><DEDENT>values = (<EOL>self._get_namespace_name_to_encoding(desc='<STR_LIT>')<EOL>if use_names else<EOL>self._get_namespace_identifier_to_encoding(desc='<STR_LIT>')<EOL>)<EOL>write_namespace(<EOL>namespace_name=self._get_namespace_name(),<EOL>namespace_keyword=self._get_namespace_keyword(),<EOL>namespace_query_url=self.identifiers_url,<EOL>values=values,<EOL>file=file,<EOL>)<EOL>


def write_bel_annotation(self, file: TextIO) -> None:


def write_bel_namespace_mappings(self, file: TextIO, **kwargs) -> None:

json.dump(self._get_namespace_identifier_to_name(**kwargs), file, indent=<NUM_LIT:2>, sort_keys=True)<EOL>


def write_directory(self, directory: str) -> bool:

current_md5_hash = self.get_namespace_hash()<EOL>md5_hash_path = os.path.join(directory, f'<STR_LIT>')<EOL>if not os.path.exists(md5_hash_path):<EOL><INDENT>old_md5_hash = None<EOL><DEDENT>else:<EOL><INDENT>with open(md5_hash_path) as file:<EOL><INDENT>old_md5_hash = file.read().strip()<EOL><DEDENT><DEDENT>if old_md5_hash == current_md5_hash:<EOL><INDENT>return False<EOL><DEDENT>with open(os.path.join(directory, f'<STR_LIT>'), '<STR_LIT:w>') as file:<EOL><INDENT>self.write_bel_namespace(file, use_names=False)<EOL><DEDENT>with open(md5_hash_path, '<STR_LIT:w>') as file:<EOL><INDENT>print(current_md5_hash, file=file)<EOL><DEDENT>if self.has_names:<EOL><INDENT>with open(os.path.join(directory, f'<STR_LIT>'), '<STR_LIT:w>') as file:<EOL><INDENT>self.write_bel_namespace(file, use_names=True)<EOL><DEDENT>with open(os.path.join(directory, f'<STR_LIT>'), '<STR_LIT:w>') as file:<EOL><INDENT>self.write_bel_namespace_mappings(file, desc='<STR_LIT>')<EOL><DEDENT><DEDENT>return True<EOL>

Write a BEL namespace for identifiers, names, name hash, and mappings to the given directory.


def get_namespace_hash(self, hash_fn=hashlib.md5) -> str:

m = hash_fn()<EOL>if self.has_names:<EOL><INDENT>items = self._get_namespace_name_to_encoding(desc='<STR_LIT>').items()<EOL><DEDENT>else:<EOL><INDENT>items = self._get_namespace_identifier_to_encoding(desc='<STR_LIT>').items()<EOL><DEDENT>for name, encoding in items:<EOL><INDENT>m.update(f'<STR_LIT>'.encode('<STR_LIT:utf8>'))<EOL><DEDENT>return m.hexdigest()<EOL>

Get the namespace hash.

        Defaults to MD5.


@staticmethod<EOL><INDENT>def _cli_add_to_bel_namespace(main: click.Group) -> click.Group:<DEDENT>

return add_cli_to_bel_namespace(main)<EOL>


@staticmethod<EOL><INDENT>def _cli_add_clear_bel_namespace(main: click.Group) -> click.Group:<DEDENT>

return add_cli_clear_bel_namespace(main)<EOL>


@staticmethod<EOL><INDENT>def _cli_add_write_bel_namespace(main: click.Group) -> click.Group:<DEDENT>

return add_cli_write_bel_namespace(main)<EOL>


@staticmethod<EOL><INDENT>def _cli_add_write_bel_annotation(main: click.Group) -> click.Group:<DEDENT>

return add_cli_write_bel_annotation(main)<EOL>