def hash_file(fpath, blocksize=<NUM_LIT>, stride=<NUM_LIT:1>, hasher=NoParam,<EOL>hashlen=NoParam, base=NoParam):

base = _rectify_base(base)<EOL>hashlen = _rectify_hashlen(hashlen)<EOL>hasher = _rectify_hasher(hasher)()<EOL>with open(fpath, '<STR_LIT:rb>') as file:<EOL><INDENT>buf = file.read(blocksize)<EOL>if stride > <NUM_LIT:1>:<EOL><INDENT>while len(buf) > <NUM_LIT:0>:<EOL><INDENT>hasher.update(buf)<EOL>file.seek(blocksize * (stride - <NUM_LIT:1>), <NUM_LIT:1>)<EOL>buf = file.read(blocksize)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while len(buf) > <NUM_LIT:0>:<EOL><INDENT>hasher.update(buf)<EOL>buf = file.read(blocksize)<EOL><DEDENT><DEDENT><DEDENT>text = _digest_hasher(hasher, hashlen, base)<EOL>return text<EOL>

Hashes the data in a file on disk.

Args:
    fpath (PathLike):  file path string

    blocksize (int): 2 ** 16. Affects speed of reading file

    stride (int): strides > 1 skip data to hash, useful for faster
                  hashing, but less accurate, also makes hash dependant on
                  blocksize.

    hasher (HASH): hash algorithm from hashlib, defaults to `sha512`.

    hashlen (int): maximum number of symbols in the returned hash. If
        not specified, all are returned.

    base (list, str): list of symbols or shorthand key. Valid keys are
        'abc', 'hex', and 'dec'. Defaults to 'hex'.

Notes:
    For better hashes keep stride = 1
    For faster hashes set stride > 1
    blocksize matters when stride > 1

References:
    http://stackoverflow.com/questions/3431825/md5-checksum-of-a-file
    http://stackoverflow.com/questions/5001893/when-to-use-sha-1-vs-sha-2

Example:
    >>> import ubelt as ub
    >>> from os.path import join
    >>> fpath = join(ub.ensure_app_cache_dir('ubelt'), 'tmp.txt')
    >>> ub.writeto(fpath, 'foobar')
    >>> print(ub.hash_file(fpath, hasher='sha1', base='hex'))
    8843d7f92416211de9ebb963ff4ce28125932878

Example:
    >>> import ubelt as ub
    >>> from os.path import join
    >>> fpath = ub.touch(join(ub.ensure_app_cache_dir('ubelt'), 'empty_file'))
    >>> # Test that the output is the same as sha1sum
    >>> if ub.find_exe('sha1sum'):
    >>>     want = ub.cmd(['sha1sum', fpath], verbose=2)['out'].split(' ')[0]
    >>>     got = ub.hash_file(fpath, hasher='sha1')
    >>>     print('want = {!r}'.format(want))
    >>>     print('got = {!r}'.format(got))
    >>>     assert want.endswith(got)
    >>> # Do the same for sha512 sum and md5sum
    >>> if ub.find_exe('sha512sum'):
    >>>     want = ub.cmd(['sha512sum', fpath], verbose=2)['out'].split(' ')[0]
    >>>     got = ub.hash_file(fpath, hasher='sha512')
    >>>     print('want = {!r}'.format(want))
    >>>     print('got = {!r}'.format(got))
    >>>     assert want.endswith(got)
    >>> if ub.find_exe('md5sum'):
    >>>     want = ub.cmd(['md5sum', fpath], verbose=2)['out'].split(' ')[0]
    >>>     got = ub.hash_file(fpath, hasher='md5')
    >>>     print('want = {!r}'.format(want))
    >>>     print('got = {!r}'.format(got))
    >>>     assert want.endswith(got)


<EOL>if not isinstance(hash_types, (list, tuple)):<EOL><INDENT>hash_types = [hash_types]<EOL><DEDENT>def _decor_closure(hash_func):<EOL><INDENT>for hash_type in hash_types:<EOL><INDENT>key = (hash_type.__module__, hash_type.__name__)<EOL>self.keyed_extensions[key] = (hash_type, hash_func)<EOL><DEDENT>return hash_func<EOL><DEDENT>return _decor_closure<EOL>

Registers a function to generate a hash for data of the appropriate
types. This can be used to register custom classes. Internally this is
used to define how to hash non-builtin objects like ndarrays and uuids.

The registered function should return a tuple of bytes. First a small
prefix hinting at the data type, and second the raw bytes that can be
hashed.

Args:
    hash_types (class or tuple of classes):

Returns:
    func: closure to be used as the decorator

Example:
    >>> # xdoctest: +SKIP
    >>> # Skip this doctest because we dont want tests to modify
    >>> # the global state.
    >>> import ubelt as ub
    >>> import pytest
    >>> class MyType(object):
    ...     def __init__(self, id):
    ...         self.id = id
    >>> data = MyType(1)
    >>> # Custom types wont work with ub.hash_data by default
    >>> with pytest.raises(TypeError):
    ...     ub.hash_data(data)
    >>> # You can register your functions with ubelt's internal
    >>> # hashable_extension registery.
    >>> @ub.util_hash._HASHABLE_EXTENSIONS.register(MyType)
    >>> def hash_my_type(data):
    ...     return b'mytype', six.b(ub.hash_data(data.id))
    >>> # TODO: allow hash_data to take an new instance of
    >>> # HashableExtensions, so we dont have to modify the global
    >>> # ubelt state when we run tests.
    >>> my_instance = MyType(1)
    >>> ub.hash_data(my_instance)


self.iterable_checks.append(func)<EOL>return func<EOL>

Registers a function that detects when a type is iterable


<EOL>query_hash_type = data.__class__<EOL>key = (query_hash_type.__module__, query_hash_type.__name__)<EOL>try:<EOL><INDENT>hash_type, hash_func = self.keyed_extensions[key]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise TypeError('<STR_LIT>' % (<EOL>query_hash_type))<EOL><DEDENT>return hash_func<EOL>

Returns an appropriate function to hash `data` if one has been
registered.

Raises:
    TypeError : if data has no registered hash methods

Example:
    >>> import ubelt as ub
    >>> import pytest
    >>> if not ub.modname_to_modpath('numpy'):
    ...     raise pytest.skip('numpy is optional')
    >>> self = HashableExtensions()
    >>> self._register_numpy_extensions()
    >>> self._register_builtin_class_extensions()

    >>> import numpy as np
    >>> data = np.array([1, 2, 3])
    >>> self.lookup(data[0])

    >>> class Foo(object):
    >>>     def __init__(f):
    >>>         f.attr = 1
    >>> data = Foo()
    >>> assert pytest.raises(TypeError, self.lookup, data)

    >>> # If ub.hash_data doesnt support your object,
    >>> # then you can register it.
    >>> @self.register(Foo)
    >>> def _hashfoo(data):
    >>>     return b'FOO', data.attr
    >>> func = self.lookup(data)
    >>> assert func(data)[1] == 1

    >>> data = uuid.uuid4()
    >>> self.lookup(data)


<EOL>import numpy as np<EOL>numpy_floating_types = (np.float16, np.float32, np.float64)<EOL>if hasattr(np, '<STR_LIT>'):  <EOL><INDENT>numpy_floating_types = numpy_floating_types + (np.float128,)<EOL><DEDENT>@self.add_iterable_check<EOL>def is_object_ndarray(data):<EOL><INDENT>return isinstance(data, np.ndarray) and data.dtype.kind == '<STR_LIT:O>'<EOL><DEDENT>@self.register(np.ndarray)<EOL>def hash_numpy_array(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>if data.dtype.kind == '<STR_LIT:O>':<EOL><INDENT>msg = '<STR_LIT>'<EOL>raise TypeError(msg)<EOL><DEDENT>else:<EOL><INDENT>header = b'<STR_LIT>'.join(_hashable_sequence((len(data.shape), data.shape)))<EOL>dtype = b'<STR_LIT>'.join(_hashable_sequence(data.dtype.descr))<EOL>hashable = header + dtype + data.tobytes()<EOL><DEDENT>prefix = b'<STR_LIT>'<EOL>return prefix, hashable<EOL><DEDENT>@self.register((np.int64, np.int32, np.int16, np.int8) +<EOL>(np.uint64, np.uint32, np.uint16, np.uint8))<EOL>def _hash_numpy_int(data):<EOL><INDENT>return _convert_to_hashable(int(data))<EOL><DEDENT>@self.register(numpy_floating_types)<EOL>def _hash_numpy_float(data):<EOL><INDENT>return _convert_to_hashable(float(data))<EOL><DEDENT>@self.register(np.random.RandomState)<EOL>def _hash_numpy_random_state(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>hashable = b'<STR_LIT>'.join(_hashable_sequence(data.get_state()))<EOL>prefix = b'<STR_LIT>'<EOL>return prefix, hashable<EOL><DEDENT>


@self.register(uuid.UUID)<EOL>def _hash_uuid(data):<EOL><INDENT>hashable = data.bytes<EOL>prefix = b'<STR_LIT>'<EOL>return prefix, hashable<EOL><DEDENT>@self.register(OrderedDict)<EOL>def _hash_ordered_dict(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>hashable = b'<STR_LIT>'.join(_hashable_sequence(list(data.items())))<EOL>prefix = b'<STR_LIT>'<EOL>return prefix, hashable<EOL><DEDENT>

Register hashing extensions for a selection of classes included in
python stdlib.

Example:
    >>> data = uuid.UUID('7e9d206b-dc02-4240-8bdb-fffe858121d0')
    >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
    cryarepd
    >>> data = OrderedDict([('a', 1), ('b', 2), ('c', [1, 2, 3]),
    >>>                     (4, OrderedDict())])
    >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
    qjspicvv

    gpxtclct


def highlight_code(text, lexer_name='<STR_LIT>', **kwargs):

<EOL>lexer_name = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:h>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:c>': '<STR_LIT>',<EOL>}.get(lexer_name.replace('<STR_LIT:.>', '<STR_LIT>'), lexer_name)<EOL>try:<EOL><INDENT>import pygments<EOL>import pygments.lexers<EOL>import pygments.formatters<EOL>import pygments.formatters.terminal<EOL>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>import colorama<EOL>colorama.init()<EOL><DEDENT>formater = pygments.formatters.terminal.TerminalFormatter(bg='<STR_LIT>')<EOL>lexer = pygments.lexers.get_lexer_by_name(lexer_name, **kwargs)<EOL>new_text = pygments.highlight(text, lexer, formater)<EOL><DEDENT>except ImportError:  <EOL><INDENT>import warnings<EOL>warnings.warn('<STR_LIT>')<EOL>new_text = text<EOL><DEDENT>return new_text<EOL>

Highlights a block of text using ANSI tags based on language syntax.

Args:
    text (str): plain text to highlight
    lexer_name (str): name of language
    **kwargs: passed to pygments.lexers.get_lexer_by_name

Returns:
    str: text : highlighted text
        If pygments is not installed, the plain text is returned.

CommandLine:
    python -c "import pygments.formatters; print(list(pygments.formatters.get_all_formatters()))"

Example:
    >>> import ubelt as ub
    >>> text = 'import ubelt as ub; print(ub)'
    >>> new_text = ub.highlight_code(text)
    >>> print(new_text)


if color is None:<EOL><INDENT>return text<EOL><DEDENT>try:<EOL><INDENT>import pygments<EOL>import pygments.console<EOL>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>import colorama<EOL>colorama.init()<EOL><DEDENT>ansi_text = pygments.console.colorize(color, text)<EOL>return ansi_text<EOL><DEDENT>except ImportError:  <EOL><INDENT>import warnings<EOL>warnings.warn('<STR_LIT>')<EOL>return text<EOL><DEDENT>

r"""
    Colorizes text a single color using ansii tags.

    Args:
        text (str): text to colorize
        color (str): may be one of the following: yellow, blink, lightgray,
            underline, darkyellow, blue, darkblue, faint, fuchsia, black,
            white, red, brown, turquoise, bold, darkred, darkgreen, reset,
            standout, darkteal, darkgray, overline, purple, green, teal, fuscia

    Returns:
        str: text : colorized text.
            If pygments is not installed plain text is returned.

    CommandLine:
        python -c "import pygments.console; print(sorted(pygments.console.codes.keys()))"
        python -m ubelt.util_colors color_text

    Example:
        >>> text = 'raw text'
        >>> import pytest
        >>> import ubelt as ub
        >>> if ub.modname_to_modpath('pygments'):
        >>>     # Colors text only if pygments is installed
        >>>     assert color_text(text, 'red') == '\x1b[31;01mraw text\x1b[39;49;00m'
        >>>     assert color_text(text, None) == 'raw text'
        >>> else:
        >>>     # Otherwise text passes through unchanged
        >>>     assert color_text(text, 'red') == 'raw text'
        >>>     assert color_text(text, None) == 'raw text'


return prefix + text.replace('<STR_LIT:\n>', '<STR_LIT:\n>' + prefix)<EOL>

Indents a block of text

Args:
    text (str): text to indent
    prefix (str): prefix to add to each line (default = '    ')

Returns:
    str: indented text

CommandLine:
    python -m util_str indent

Example:
    >>> from ubelt.util_str import *  # NOQA
    >>> NL = chr(10)  # newline character
    >>> text = 'Lorem ipsum' + NL + 'dolor sit amet'
    >>> prefix = '    '
    >>> result = indent(text, prefix)
    >>> assert all(t.startswith(prefix) for t in result.split(NL))


import textwrap  <EOL>return textwrap.dedent(block_str).strip('<STR_LIT:\n>')<EOL>

Wraps multiline string blocks and returns unindented code.
Useful for templated code defined in indented parts of code.

Args:
    block_str (str): typically in the form of a multiline string

Returns:
    str: the unindented string

CommandLine:
    python -m ubelt.util_str codeblock

Example:
    >>> from ubelt.util_str import *  # NOQA
    >>> # Simulate an indented part of code
    >>> if True:
    >>>     # notice the indentation on this will be normal
    >>>     codeblock_version = codeblock(
    ...             '''
    ...             def foo():
    ...                 return 'bar'
    ...             '''
    ...         )
    >>>     # notice the indentation and newlines on this will be odd
    >>>     normal_version = ('''
    ...         def foo():
    ...             return 'bar'
    ...     ''')
    >>> assert normal_version != codeblock_version
    >>> print('Without codeblock')
    >>> print(normal_version)
    >>> print('With codeblock')
    >>> print(codeblock_version)


import unicodedata<EOL>if '<STR_LIT:\n>' in sep or '<STR_LIT:\r>' in sep:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>args = [unicodedata.normalize('<STR_LIT>', ensure_unicode(val)) for val in args]<EOL>arglines = [a.split('<STR_LIT:\n>') for a in args]<EOL>height = max(map(len, arglines))<EOL>arglines = [lines + ['<STR_LIT>'] * (height - len(lines)) for lines in arglines]<EOL>all_lines = ['<STR_LIT>' for _ in range(height)]<EOL>width = <NUM_LIT:0><EOL>n_args = len(args)<EOL>for sx, lines in enumerate(arglines):<EOL><INDENT>for lx, line in enumerate(lines):<EOL><INDENT>all_lines[lx] += line<EOL><DEDENT>width = max(width, max(map(len, all_lines)))<EOL>if sx < n_args - <NUM_LIT:1>:<EOL><INDENT>for lx, line in list(enumerate(all_lines)):<EOL><INDENT>residual = width - len(line)<EOL>all_lines[lx] = line + ('<STR_LIT:U+0020>' * residual) + sep<EOL><DEDENT>width += len(sep)<EOL><DEDENT><DEDENT>all_lines = [line.rstrip('<STR_LIT:U+0020>') for line in all_lines]<EOL>ret = '<STR_LIT:\n>'.join(all_lines)<EOL>return ret<EOL>

Horizontally concatenates strings preserving indentation

Concatenates a list of objects ensuring that the next item in the list is
all the way to the right of any previous items.

Args:
    args (List[str]): strings to concatenate
    sep (str): separator (defaults to '')

CommandLine:
    python -m ubelt.util_str hzcat

Example1:
    >>> import ubelt as ub
    >>> B = ub.repr2([[1, 2], [3, 457]], nl=1, cbr=True, trailsep=False)
    >>> C = ub.repr2([[5, 6], [7, 8]], nl=1, cbr=True, trailsep=False)
    >>> args = ['A = ', B, ' * ', C]
    >>> print(ub.hzcat(args))
    A = [[1, 2],   * [[5, 6],
         [3, 457]]    [7, 8]]

Example2:
    >>> from ubelt.util_str import *
    >>> import ubelt as ub
    >>> import unicodedata
    >>> aa = unicodedata.normalize('NFD', 'á')  # a unicode char with len2
    >>> B = ub.repr2([['θ', aa], [aa, aa, aa]], nl=1, si=True, cbr=True, trailsep=False)
    >>> C = ub.repr2([[5, 6], [7, 'θ']], nl=1, si=True, cbr=True, trailsep=False)
    >>> args = ['A', '=', B, '*', C]
    >>> print(ub.hzcat(args, sep='｜'))
    A｜=｜[[θ, á],   ｜*｜[[5, 6],
     ｜ ｜ [á, á, á]]｜ ｜ [7, θ]]


if isinstance(text, six.text_type):<EOL><INDENT>return text<EOL><DEDENT>elif isinstance(text, six.binary_type):<EOL><INDENT>return text.decode('<STR_LIT:utf8>')<EOL><DEDENT>else:  <EOL><INDENT>raise ValueError('<STR_LIT>'.format(text))<EOL><DEDENT>

r"""
    Casts bytes into utf8 (mostly for python2 compatibility)

    References:
        http://stackoverflow.com/questions/12561063/extract-data-from-file

    Example:
        >>> from ubelt.util_str import *
        >>> import codecs  # NOQA
        >>> assert ensure_unicode('my ünicôdé strįng') == 'my ünicôdé strįng'
        >>> assert ensure_unicode('text1') == 'text1'
        >>> assert ensure_unicode('text1'.encode('utf8')) == 'text1'
        >>> assert ensure_unicode('ï»¿text1'.encode('utf8')) == 'ï»¿text1'
        >>> assert (codecs.BOM_UTF8 + 'text»¿'.encode('utf8')).decode('utf8')


custom_extensions = kwargs.get('<STR_LIT>', None)<EOL>_return_info = kwargs.get('<STR_LIT>', False)<EOL>kwargs['<STR_LIT>'] = _rectify_root_info(kwargs.get('<STR_LIT>', None))<EOL>outstr = None<EOL>_leaf_info = None<EOL>if custom_extensions:<EOL><INDENT>func = custom_extensions.lookup(data)<EOL>if func is not None:<EOL><INDENT>outstr = func(data, **kwargs)<EOL><DEDENT><DEDENT>if outstr is None:<EOL><INDENT>if isinstance(data, dict):<EOL><INDENT>outstr, _leaf_info = _format_dict(data, **kwargs)<EOL><DEDENT>elif isinstance(data, (list, tuple, set, frozenset)):<EOL><INDENT>outstr, _leaf_info = _format_list(data, **kwargs)<EOL><DEDENT><DEDENT>if outstr is None:<EOL><INDENT>func = _FORMATTER_EXTENSIONS.lookup(data)<EOL>if func is not None:<EOL><INDENT>outstr = func(data, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>outstr = _format_object(data, **kwargs)<EOL><DEDENT><DEDENT>if _return_info:<EOL><INDENT>_leaf_info = _rectify_leaf_info(_leaf_info)<EOL>return outstr, _leaf_info<EOL><DEDENT>else:<EOL><INDENT>return outstr<EOL><DEDENT>

Makes a pretty and easy-to-doctest string representation!

This is an alternative to repr, and `pprint.pformat` that attempts to be
both more configurable and generate output that is consistent between
python versions.

Notes:
    This function has many keyword arguments that can be used to customize
    the final representation. For convinience some of the more frequently
    used kwargs have short aliases. See `Args` for more details.

Args:
    data (object): an arbitrary python object
    **kwargs: see `the Kwargs` section

Kwargs:
    si, stritems, (bool):
        dict/list items use str instead of repr

    strkeys, sk (bool):
        dict keys use str instead of repr

    strvals, sv (bool):
        dict values use str instead of repr

    nl, newlines (int | bool):
        number of top level nestings to place a newline after. If true all
        items are followed by newlines regardless of nesting level.
        Defaults to 1 for lists and True for dicts.

    nobr, nobraces (bool, default=False):
        if True, text will not contain outer braces for containers

    cbr, compact_brace (bool, default=False):
        if True, braces are compactified (i.e. they will not have newlines
        placed directly after them, think java / K&R / 1TBS)

    trailsep, trailing_sep (bool):
        if True, a separator is placed after the last item in a sequence.
        By default this is True if there are any `nl > 0`.

    explicit (bool, default=False):
        changes dict representation from `{k1: v1, ...}` to
        `dict(k1=v1, ...)`.

    precision (int, default=None):
        if specified floats are formatted with this precision

    kvsep (str, default=': '):
        separator between keys and values

    itemsep (str, default=' '):
        separator between items

    sort (bool):
        if True, attempts to sort all unordered collections in the returned
        text. NOTE: currently if True this will sort lists, this may not be
        a correct thing to do, as such the behavior of this arg is subject
        to change.

    suppress_small (bool):
        passed to `numpy.array2string` for ndarrays

    max_line_width (int):
        passed to `numpy.array2string` for ndarrays

    with_dtype (bool):
        only relevant to ndarrays. if True includes the dtype.

Returns:
    str: outstr: output string

Notes:
    There are also internal kwargs, which should not be used:
        _return_info (bool):  return information about child context
        _root_info (depth): information about parent context

CommandLine:
    python -m ubelt.util_format repr2:0
    python -m ubelt.util_format repr2:1

Example:
    >>> from ubelt.util_format import *
    >>> import ubelt as ub
    >>> dict_ = {
    ...     'custom_types': [slice(0, 1, None), 1/3],
    ...     'nest_dict': {'k1': [1, 2, {3: {4, 5}}],
    ...                   'key2': [1, 2, {3: {4, 5}}],
    ...                   'key3': [1, 2, {3: {4, 5}}],
    ...                   },
    ...     'nest_dict2': {'k': [1, 2, {3: {4, 5}}]},
    ...     'nested_tuples': [tuple([1]), tuple([2, 3]), frozenset([4, 5, 6])],
    ...     'one_tup': tuple([1]),
    ...     'simple_dict': {'spam': 'eggs', 'ham': 'jam'},
    ...     'simple_list': [1, 2, 'red', 'blue'],
    ...     'odict': ub.odict([(1, '1'), (2, '2')]),
    ... }
    >>> result = repr2(dict_, nl=3, precision=2); print(result)
    >>> result = repr2(dict_, nl=2, precision=2); print(result)
    >>> result = repr2(dict_, nl=1, precision=2); print(result)
    >>> result = repr2(dict_, nl=1, precision=2, itemsep='', explicit=True); print(result)
    >>> result = repr2(dict_, nl=1, precision=2, nobr=1, itemsep='', explicit=True); print(result)
    >>> result = repr2(dict_, nl=3, precision=2, cbr=True); print(result)
    >>> result = repr2(dict_, nl=3, precision=2, si=True); print(result)
    >>> result = repr2(dict_, nl=3, sort=True); print(result)
    >>> result = repr2(dict_, nl=3, sort=False, trailing_sep=False); print(result)
    >>> result = repr2(dict_, nl=3, sort=False, trailing_sep=False, nobr=True); print(result)

Example:
    >>> from ubelt.util_format import *
    >>> def _nest(d, w):
    ...     if d == 0:
    ...         return {}
    ...     else:
    ...         return {'n{}'.format(d): _nest(d - 1, w + 1), 'm{}'.format(d): _nest(d - 1, w + 1)}
    >>> dict_ = _nest(d=4, w=1)
    >>> result = repr2(dict_, nl=6, precision=2, cbr=1)
    >>> print('---')
    >>> print(result)
    >>> result = repr2(dict_, nl=-1, precision=2)
    >>> print('---')
    >>> print(result)


kwargs['<STR_LIT>'] = _rectify_root_info(kwargs.get('<STR_LIT>', None))<EOL>kwargs['<STR_LIT>']['<STR_LIT>'] += <NUM_LIT:1><EOL>newlines = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', <NUM_LIT:1>))<EOL>kwargs['<STR_LIT>'] = _rectify_countdown_or_bool(newlines)<EOL>nobraces = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', False))<EOL>itemsep = kwargs.get('<STR_LIT>', '<STR_LIT:U+0020>')<EOL>compact_brace = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', False))<EOL>itemstrs, _leaf_info = _list_itemstrs(list_, **kwargs)<EOL>if len(itemstrs) == <NUM_LIT:0>:<EOL><INDENT>nobraces = False  <EOL><DEDENT>is_tuple = isinstance(list_, tuple)<EOL>is_set = isinstance(list_, (set, frozenset,))<EOL>if nobraces:<EOL><INDENT>lbr, rbr = '<STR_LIT>', '<STR_LIT>'<EOL><DEDENT>elif is_tuple:<EOL><INDENT>lbr, rbr  = '<STR_LIT:(>', '<STR_LIT:)>'<EOL><DEDENT>elif is_set:<EOL><INDENT>lbr, rbr  = '<STR_LIT:{>', '<STR_LIT:}>'<EOL><DEDENT>else:<EOL><INDENT>lbr, rbr  = '<STR_LIT:[>', '<STR_LIT:]>'<EOL><DEDENT>trailing_sep = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', newlines > <NUM_LIT:0> and len(itemstrs)))<EOL>if is_tuple and len(list_) <= <NUM_LIT:1>:<EOL><INDENT>trailing_sep = True<EOL><DEDENT>if len(itemstrs) == <NUM_LIT:0>:<EOL><INDENT>newlines = False<EOL><DEDENT>retstr = _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,<EOL>trailing_sep, compact_brace, lbr, rbr)<EOL>return retstr, _leaf_info<EOL>

Makes a pretty printable / human-readable string representation of a
sequence. In most cases this string could be evaled.

Args:
    list_ (list): input list
    **kwargs: nl, newlines, packed, nobr, nobraces, itemsep, trailing_sep,
        strvals indent_, precision, use_numpy, with_dtype, force_dtype,
        stritems, strkeys, explicit, sort, key_order, maxlen

Returns:
    Tuple[str, Dict] : retstr, _leaf_info

Example:
    >>> print(_format_list([])[0])
    []
    >>> print(_format_list([], nobr=True)[0])
    []
    >>> print(_format_list([1], nl=0)[0])
    [1]
    >>> print(_format_list([1], nobr=True)[0])
    1,


kwargs['<STR_LIT>'] = _rectify_root_info(kwargs.get('<STR_LIT>', None))<EOL>kwargs['<STR_LIT>']['<STR_LIT>'] += <NUM_LIT:1><EOL>stritems = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', False))<EOL>if stritems:<EOL><INDENT>kwargs['<STR_LIT>'] = True<EOL>kwargs['<STR_LIT>'] = True<EOL><DEDENT>kwargs['<STR_LIT>'] = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', False))<EOL>kwargs['<STR_LIT>'] = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', False))<EOL>newlines = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', True))<EOL>kwargs['<STR_LIT>'] = _rectify_countdown_or_bool(newlines)<EOL>nobraces = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', False))<EOL>compact_brace = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', False))<EOL>trailing_sep = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', newlines > <NUM_LIT:0>))<EOL>explicit = kwargs.get('<STR_LIT>', False)<EOL>itemsep = kwargs.get('<STR_LIT>', '<STR_LIT:U+0020>')<EOL>if len(dict_) == <NUM_LIT:0>:<EOL><INDENT>retstr = '<STR_LIT>' if explicit else '<STR_LIT:{}>'<EOL>_leaf_info = None<EOL><DEDENT>else:<EOL><INDENT>itemstrs, _leaf_info = _dict_itemstrs(dict_, **kwargs)<EOL>if nobraces:<EOL><INDENT>lbr, rbr = '<STR_LIT>', '<STR_LIT>'<EOL><DEDENT>elif explicit:<EOL><INDENT>lbr, rbr = '<STR_LIT>', '<STR_LIT:)>'<EOL><DEDENT>else:<EOL><INDENT>lbr, rbr = '<STR_LIT:{>', '<STR_LIT:}>'<EOL><DEDENT>retstr = _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,<EOL>trailing_sep, compact_brace, lbr, rbr)<EOL><DEDENT>return retstr, _leaf_info<EOL>

Makes a pretty printable / human-readable string representation of a
dictionary. In most cases this string could be evaled.

Args:
    dict_ (dict):  a dictionary
    **kwargs: si, stritems, strkeys, strvals, sk, sv, nl, newlines, nobr,
              nobraces, cbr, compact_brace, trailing_sep,
              explicit, itemsep, precision, kvsep, sort

Returns:
    Tuple[str, Dict] : retstr, _leaf_info

Kwargs:
    sort (None): if True, sorts ALL collections and subcollections,
        note, collections with undefined orders (e.g. dicts, sets) are
        sorted by default. (default = None)
    nl (int): preferred alias for newline. can be a countdown variable
        (default = None)
    explicit (int): can be a countdown variable. if True, uses
        dict(a=b) syntax instead of {'a': b}
    nobr (bool): removes outer braces (default = False)


def _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,<EOL>trailing_sep, compact_brace, lbr, rbr):

<EOL>use_newline = newlines > <NUM_LIT:0><EOL>if newlines < <NUM_LIT:0>:<EOL><INDENT>use_newline = (-newlines) < _leaf_info['<STR_LIT>']<EOL><DEDENT>if use_newline:<EOL><INDENT>sep = '<STR_LIT>'<EOL>if nobraces:<EOL><INDENT>body_str = sep.join(itemstrs)<EOL>if trailing_sep and len(itemstrs) > <NUM_LIT:0>:<EOL><INDENT>body_str += '<STR_LIT:U+002C>'<EOL><DEDENT>retstr = body_str<EOL><DEDENT>else:<EOL><INDENT>if compact_brace:<EOL><INDENT>indented = itemstrs<EOL><DEDENT>else:<EOL><INDENT>import ubelt as ub<EOL>prefix = '<STR_LIT:U+0020>' * <NUM_LIT:4><EOL>indented = [ub.indent(s, prefix) for s in itemstrs]<EOL><DEDENT>body_str = sep.join(indented)<EOL>if trailing_sep and len(itemstrs) > <NUM_LIT:0>:<EOL><INDENT>body_str += '<STR_LIT:U+002C>'<EOL><DEDENT>if compact_brace:<EOL><INDENT>braced_body_str = (lbr + body_str.replace('<STR_LIT:\n>', '<STR_LIT>') + rbr)<EOL><DEDENT>else:<EOL><INDENT>braced_body_str = (lbr + '<STR_LIT:\n>' + body_str + '<STR_LIT:\n>' + rbr)<EOL><DEDENT>retstr = braced_body_str<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sep = '<STR_LIT:U+002C>' + itemsep<EOL>body_str = sep.join(itemstrs)<EOL>if trailing_sep and len(itemstrs) > <NUM_LIT:0>:<EOL><INDENT>body_str += '<STR_LIT:U+002C>'<EOL><DEDENT>retstr  = (lbr + body_str +  rbr)<EOL><DEDENT>return retstr<EOL>

Joins string-ified items with separators newlines and container-braces.


import ubelt as ub<EOL>explicit = kwargs.get('<STR_LIT>', False)<EOL>kwargs['<STR_LIT>'] = _rectify_countdown_or_bool(explicit)<EOL>precision = kwargs.get('<STR_LIT>', None)<EOL>kvsep = kwargs.get('<STR_LIT>', '<STR_LIT>')<EOL>if explicit:<EOL><INDENT>kvsep = '<STR_LIT:=>'<EOL><DEDENT>def make_item_str(key, val):<EOL><INDENT>if explicit or kwargs.get('<STR_LIT>', False):<EOL><INDENT>key_str = six.text_type(key)<EOL><DEDENT>else:<EOL><INDENT>key_str = repr2(key, precision=precision, newlines=<NUM_LIT:0>)<EOL><DEDENT>prefix = key_str + kvsep<EOL>kwargs['<STR_LIT>'] = True<EOL>val_str, _leaf_info = repr2(val, **kwargs)<EOL>pos = val_str.find('<STR_LIT:\n>')<EOL>first_line = val_str if pos == -<NUM_LIT:1> else val_str[:pos]<EOL>compact_brace = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', False))<EOL>if compact_brace or not first_line.rstrip().endswith(tuple('<STR_LIT>')):<EOL><INDENT>rest = '<STR_LIT>' if pos == -<NUM_LIT:1> else val_str[pos:]<EOL>val_str = first_line.lstrip() + rest<EOL>if '<STR_LIT:\n>' in prefix:<EOL><INDENT>item_str = prefix + val_str<EOL><DEDENT>else:<EOL><INDENT>item_str = ub.hzcat([prefix, val_str])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>item_str = prefix + val_str<EOL><DEDENT>return item_str, _leaf_info<EOL><DEDENT>items = list(six.iteritems(dict_))<EOL>_tups = [make_item_str(key, val) for (key, val) in items]<EOL>itemstrs = [t[<NUM_LIT:0>] for t in _tups]<EOL>max_height = max([t[<NUM_LIT:1>]['<STR_LIT>'] for t in _tups]) if _tups else <NUM_LIT:0><EOL>_leaf_info = {<EOL>'<STR_LIT>': max_height + <NUM_LIT:1>,<EOL>}<EOL>sort = kwargs.get('<STR_LIT>', None)<EOL>if sort is None:<EOL><INDENT>sort = True<EOL><DEDENT>if isinstance(dict_, collections.OrderedDict):<EOL><INDENT>sort = False<EOL><DEDENT>if sort:<EOL><INDENT>itemstrs = _sort_itemstrs(items, itemstrs)<EOL><DEDENT>return itemstrs, _leaf_info<EOL>

Create a string representation for each item in a dict.

Example:
    >>> from ubelt.util_format import *
    >>> dict_ =  {'b': .1, 'l': 'st', 'g': 1.0, 's': 10, 'm': 0.9, 'w': .5}
    >>> kwargs = {'strkeys': True}
    >>> itemstrs, _ = _dict_itemstrs(dict_, **kwargs)
    >>> char_order = [p[0] for p in itemstrs]
    >>> assert char_order == ['b', 'g', 'l', 'm', 's', 'w']


items = list(list_)<EOL>kwargs['<STR_LIT>'] = True<EOL>_tups = [repr2(item, **kwargs) for item in items]<EOL>itemstrs = [t[<NUM_LIT:0>] for t in _tups]<EOL>max_height = max([t[<NUM_LIT:1>]['<STR_LIT>'] for t in _tups]) if _tups else <NUM_LIT:0><EOL>_leaf_info = {<EOL>'<STR_LIT>': max_height + <NUM_LIT:1>,<EOL>}<EOL>sort = kwargs.get('<STR_LIT>', None)<EOL>if sort is None:<EOL><INDENT>sort = isinstance(list_, (set, frozenset))<EOL><DEDENT>if sort:<EOL><INDENT>itemstrs = _sort_itemstrs(items, itemstrs)<EOL><DEDENT>return itemstrs, _leaf_info<EOL>

Create a string representation for each item in a list.


<EOL>import ubelt as ub<EOL>try:<EOL><INDENT>if _peek_isinstance(items, (set, frozenset)):<EOL><INDENT>raise TypeError<EOL><DEDENT>sortx = ub.argsort(items)<EOL><DEDENT>except TypeError:<EOL><INDENT>sortx = ub.argsort(itemstrs)<EOL><DEDENT>itemstrs = [itemstrs[x] for x in sortx]<EOL>return itemstrs<EOL>

Equivalent to `sorted(items)` except if `items` are unorderable, then
string values are used to define an ordering.


if count_or_bool is True or count_or_bool is False:<EOL><INDENT>count_or_bool_ = count_or_bool<EOL><DEDENT>elif isinstance(count_or_bool, int):<EOL><INDENT>if count_or_bool == <NUM_LIT:0>:<EOL><INDENT>return <NUM_LIT:0><EOL><DEDENT>elif count_or_bool > <NUM_LIT:0>:<EOL><INDENT>count_or_bool_ = count_or_bool - <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>count_or_bool_ = count_or_bool<EOL><DEDENT><DEDENT>else:<EOL><INDENT>count_or_bool_ = False<EOL><DEDENT>return count_or_bool_<EOL>

used by recursive functions to specify which level to turn a bool on in
counting down yields True, True, ..., False
counting up yields False, False, False, ... True

Args:
    count_or_bool (bool or int): if positive and an integer, it will count
        down, otherwise it will remain the same.

Returns:
    int or bool: count_or_bool_

CommandLine:
    python -m utool.util_str --test-_rectify_countdown_or_bool

Example:
    >>> from ubelt.util_format import _rectify_countdown_or_bool  # NOQA
    >>> count_or_bool = True
    >>> a1 = (_rectify_countdown_or_bool(2))
    >>> a2 = (_rectify_countdown_or_bool(1))
    >>> a3 = (_rectify_countdown_or_bool(0))
    >>> a4 = (_rectify_countdown_or_bool(-1))
    >>> a5 = (_rectify_countdown_or_bool(-2))
    >>> a6 = (_rectify_countdown_or_bool(True))
    >>> a7 = (_rectify_countdown_or_bool(False))
    >>> a8 = (_rectify_countdown_or_bool(None))
    >>> result = [a1, a2, a3, a4, a5, a6, a7, a8]
    >>> print(result)
    [1, 0, 0, -1, -2, True, False, False]


def _decorator(func):<EOL><INDENT>if isinstance(type, tuple):<EOL><INDENT>for t in type:<EOL><INDENT>self.func_registry[t] = func<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.func_registry[type] = func<EOL><DEDENT>return func<EOL><DEDENT>return _decorator<EOL>

Registers a custom formatting function with ub.repr2


for func in self.lazy_init:<EOL><INDENT>func()<EOL><DEDENT>for type, func in self.func_registry.items():<EOL><INDENT>if isinstance(data, type):<EOL><INDENT>return func<EOL><DEDENT><DEDENT>

Returns an appropriate function to format `data` if one has been
registered.


import numpy as np<EOL>@self.register(np.ndarray)<EOL>def format_ndarray(data, **kwargs):<EOL><INDENT>import re<EOL>strvals = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', False))<EOL>itemsep = kwargs.get('<STR_LIT>', '<STR_LIT:U+0020>')<EOL>precision = kwargs.get('<STR_LIT>', None)<EOL>suppress_small = kwargs.get('<STR_LIT>', None)<EOL>max_line_width = kwargs.get('<STR_LIT>', None)<EOL>with_dtype = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', not strvals))<EOL>newlines = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', <NUM_LIT:1>))<EOL>separator = '<STR_LIT:U+002C>' + itemsep<EOL>if strvals:<EOL><INDENT>prefix = '<STR_LIT>'<EOL>suffix = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>modname = type(data).__module__<EOL>np_nice = '<STR_LIT>'<EOL>modname = re.sub('<STR_LIT>', np_nice, modname)<EOL>modname = re.sub('<STR_LIT>', '<STR_LIT>', modname)<EOL>class_name = type(data).__name__<EOL>if class_name == '<STR_LIT>':<EOL><INDENT>class_name = '<STR_LIT>'<EOL><DEDENT>prefix = modname + '<STR_LIT:.>' + class_name + '<STR_LIT:(>'<EOL>if with_dtype:<EOL><INDENT>dtype_repr = data.dtype.name<EOL>suffix = '<STR_LIT>'.format(itemsep, np_nice, dtype_repr)<EOL><DEDENT>else:<EOL><INDENT>suffix = '<STR_LIT:)>'<EOL><DEDENT><DEDENT>if not strvals and data.size == <NUM_LIT:0> and data.shape != (<NUM_LIT:0>,):<EOL><INDENT>prefix = modname + '<STR_LIT>'<EOL>body = repr(tuple(map(int, data.shape)))<EOL><DEDENT>else:<EOL><INDENT>body = np.array2string(data, precision=precision,<EOL>separator=separator,<EOL>suppress_small=suppress_small,<EOL>prefix=prefix,<EOL>max_line_width=max_line_width)<EOL><DEDENT>if not newlines:<EOL><INDENT>body = re.sub('<STR_LIT>', '<STR_LIT>', body)<EOL><DEDENT>formatted = prefix + body + suffix<EOL>return formatted<EOL><DEDENT>self.register(np.float32)(self.func_registry[float])<EOL>

CommandLine:
    python -m ubelt.util_format FormatterExtensions._register_numpy_extensions

Example:
    >>> import sys
    >>> import pytest
    >>> import ubelt as ub
    >>> if not ub.modname_to_modpath('numpy'):
    ...     raise pytest.skip()
    >>> # xdoctest: +IGNORE_WHITESPACE
    >>> import numpy as np
    >>> data = np.array([[.2, 42, 5], [21.2, 3, .4]])
    >>> print(ub.repr2(data))
    np.array([[ 0.2, 42. ,  5. ],
              [21.2,  3. ,  0.4]], dtype=np.float64)
    >>> print(ub.repr2(data, with_dtype=False))
    np.array([[ 0.2, 42. ,  5. ],
              [21.2,  3. ,  0.4]])
    >>> print(ub.repr2(data, strvals=True))
    [[ 0.2, 42. ,  5. ],
     [21.2,  3. ,  0.4]]
    >>> data = np.empty((0, 10), dtype=np.float64)
    >>> print(ub.repr2(data, strvals=False))
    np.empty((0, 10), dtype=np.float64)
    >>> print(ub.repr2(data, strvals=True))
    []
    >>> data = np.ma.empty((0, 10), dtype=np.float64)
    >>> print(ub.repr2(data, strvals=False))
    np.ma.empty((0, 10), dtype=np.float64)


try:<EOL><INDENT>hash(item)<EOL><DEDENT>except TypeError:<EOL><INDENT>return util_hash.hash_data(item)<EOL><DEDENT>else:<EOL><INDENT>return item<EOL><DEDENT>

Returns the item if it is naturally hashable, otherwise it tries to use
ub.hash_data to make it hashable. Errors if it cannot.


kwitems = kwargs.items()<EOL>if (sys.version_info.major, sys.version_info.minor) < (<NUM_LIT:3>, <NUM_LIT:7>):  <EOL><INDENT>kwitems = sorted(kwitems)<EOL><DEDENT>kwitems = tuple(kwitems)<EOL>try:<EOL><INDENT>key = _hashable(args), _hashable(kwitems)<EOL><DEDENT>except TypeError:<EOL><INDENT>raise TypeError('<STR_LIT>'.format(args, kwargs))<EOL><DEDENT>return key<EOL>

Transforms function args into a key that can be used by the cache

CommandLine:
    xdoctest -m ubelt.util_memoize _make_signature_key

Example:
    >>> args = (4, [1, 2])
    >>> kwargs = {'a': 'b'}
    >>> key = _make_signature_key(args, kwargs)
    >>> print('key = {!r}'.format(key))
    >>> # Some mutable types cannot be handled by ub.hash_data
    >>> import pytest
    >>> import six
    >>> if six.PY2:
    >>>     import collections as abc
    >>> else:
    >>>     from collections import abc
    >>> with pytest.raises(TypeError):
    >>>     _make_signature_key((4, [1, 2], {1: 2, 'a': 'b'}), kwargs={})
    >>> class Dummy(abc.MutableSet):
    >>>     def __contains__(self, item): return None
    >>>     def __iter__(self): return iter([])
    >>>     def __len__(self): return 0
    >>>     def add(self, item, loc): return None
    >>>     def discard(self, item): return None
    >>> with pytest.raises(TypeError):
    >>>     _make_signature_key((Dummy(),), kwargs={})


cache = {}<EOL>@functools.wraps(func)<EOL>def memoizer(*args, **kwargs):<EOL><INDENT>key = _make_signature_key(args, kwargs)<EOL>if key not in cache:<EOL><INDENT>cache[key] = func(*args, **kwargs)<EOL><DEDENT>return cache[key]<EOL><DEDENT>memoizer.cache = cache<EOL>return memoizer<EOL>

memoization decorator that respects args and kwargs

References:
    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize

Args:
    func (Callable): live python function

Returns:
    func: memoized wrapper

CommandLine:
    xdoctest -m ubelt.util_memoize memoize

Example:
    >>> import ubelt as ub
    >>> closure = {'a': 'b', 'c': 'd'}
    >>> incr = [0]
    >>> def foo(key):
    >>>     value = closure[key]
    >>>     incr[0] += 1
    >>>     return value
    >>> foo_memo = ub.memoize(foo)
    >>> assert foo('a') == 'b' and foo('c') == 'd'
    >>> assert incr[0] == 2
    >>> print('Call memoized version')
    >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
    >>> assert incr[0] == 4
    >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
    >>> print('Counter should no longer increase')
    >>> assert incr[0] == 4
    >>> print('Closure changes result without memoization')
    >>> closure = {'a': 0, 'c': 1}
    >>> assert foo('a') == 0 and foo('c') == 1
    >>> assert incr[0] == 6
    >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'


<EOL>while hasattr(fget, '<STR_LIT>'):<EOL><INDENT>fget = fget.fget<EOL><DEDENT>attr_name = '<STR_LIT:_>' + fget.__name__<EOL>@functools.wraps(fget)<EOL>def fget_memoized(self):<EOL><INDENT>if not hasattr(self, attr_name):<EOL><INDENT>setattr(self, attr_name, fget(self))<EOL><DEDENT>return getattr(self, attr_name)<EOL><DEDENT>return property(fget_memoized)<EOL>

Return a property attribute for new-style classes that only calls its
getter on the first access. The result is stored and on subsequent accesses
is returned, preventing the need to call the getter any more.

This decorator can either be used by itself or by decorating another
property. In either case the method will always become a property.

Notes:
    implementation is a modified version of [1].

References:
    ..[1] https://github.com/estebistec/python-memoized-property

CommandLine:
    xdoctest -m ubelt.util_memoize memoize_property

Example:
    >>> class C(object):
    ...     load_name_count = 0
    ...     @memoize_property
    ...     def name(self):
    ...         "name's docstring"
    ...         self.load_name_count += 1
    ...         return "the name"
    ...     @memoize_property
    ...     @property
    ...     def another_name(self):
    ...         "name's docstring"
    ...         self.load_name_count += 1
    ...         return "the name"
    >>> c = C()
    >>> c.load_name_count
    0
    >>> c.name
    'the name'
    >>> c.load_name_count
    1
    >>> c.name
    'the name'
    >>> c.load_name_count
    1
    >>> c.another_name


self._instance = instance<EOL>return self<EOL>

Descriptor get method. Called when the decorated method is accessed
from an object instance.

Args:
    instance (object): the instance of the class with the memoized method
    cls (type): the type of the instance


cache = self._instance.__dict__.setdefault(self._cache_name, {})<EOL>key = _make_signature_key(args, kwargs)<EOL>if key in cache:<EOL><INDENT>return cache[key]<EOL><DEDENT>else:<EOL><INDENT>value = cache[key] = self._func(self._instance, *args, **kwargs)<EOL>return value<EOL><DEDENT>


import os<EOL>if not os.path.exists(modpath):<EOL><INDENT>import re<EOL>import zipimport<EOL>pat = '<STR_LIT>' + re.escape(os.path.sep) + '<STR_LIT>'<EOL>parts = re.split(pat, modpath, flags=re.IGNORECASE)<EOL>if len(parts) > <NUM_LIT:2>:<EOL><INDENT>archivepath = '<STR_LIT>'.join(parts[:-<NUM_LIT:1>])[:-<NUM_LIT:1>]<EOL>internal = parts[-<NUM_LIT:1>]<EOL>modname = os.path.splitext(internal)[<NUM_LIT:0>]<EOL>modname = os.path.normpath(modname)<EOL>if os.path.exists(archivepath):<EOL><INDENT>zimp_file = zipimport.zipimporter(archivepath)<EOL>module = zimp_file.load_module(modname)<EOL>return module<EOL><DEDENT><DEDENT>raise IOError('<STR_LIT>'.format(modpath))<EOL><DEDENT>else:<EOL><INDENT>module = _custom_import_modpath(modpath)<EOL>return module<EOL><DEDENT>

Imports a module via its path

Args:
    modpath (PathLike): path to the module on disk or within a zipfile.

Returns:
    module: the imported module

References:
    https://stackoverflow.com/questions/67631/import-module-given-path

Notes:
    If the module is part of a package, the package will be imported first.
    These modules may cause problems when reloading via IPython magic

    This can import a module from within a zipfile. To do this modpath
    should specify the path to the zipfile and the path to the module
    within that zipfile separated by a colon or pathsep.
    E.g. `/path/to/archive.zip:mymodule.py`

Warning:
    It is best to use this with paths that will not conflict with
    previously existing modules.

    If the modpath conflicts with a previously existing module name. And
    the target module does imports of its own relative to this conflicting
    path. In this case, the module that was loaded first will win.

    For example if you try to import '/foo/bar/pkg/mod.py' from the folder
    structure:
      - foo/
        +- bar/
           +- pkg/
              +  __init__.py
              |- mod.py
              |- helper.py

   If there exists another module named `pkg` already in sys.modules
   and mod.py does something like `from . import helper`, Python will
   assume helper belongs to the `pkg` module already in sys.modules.
   This can cause a NameError or worse --- a incorrect helper module.

Example:
    >>> import xdoctest
    >>> modpath = xdoctest.__file__
    >>> module = import_module_from_path(modpath)
    >>> assert module is xdoctest

Example:
    >>> # Test importing a module from within a zipfile
    >>> import zipfile
    >>> from xdoctest import utils
    >>> from os.path import join, expanduser
    >>> dpath = expanduser('~/.cache/xdoctest')
    >>> dpath = utils.ensuredir(dpath)
    >>> #dpath = utils.TempDir().ensure()
    >>> # Write to an external module named bar
    >>> external_modpath = join(dpath, 'bar.py')
    >>> open(external_modpath, 'w').write('testvar = 1')
    >>> internal = 'folder/bar.py'
    >>> # Move the external bar module into a zipfile
    >>> zippath = join(dpath, 'myzip.zip')
    >>> with zipfile.ZipFile(zippath, 'w') as myzip:
    >>>     myzip.write(external_modpath, internal)
    >>> # Import the bar module from within the zipfile
    >>> modpath = zippath + ':' + internal
    >>> modpath = zippath + os.path.sep + internal
    >>> module = import_module_from_path(modpath)
    >>> assert module.__name__ == os.path.normpath('folder/bar')
    >>> assert module.testvar == 1

Doctest:
    >>> import pytest
    >>> with pytest.raises(IOError):
    >>>     import_module_from_path('does-not-exist')
    >>> with pytest.raises(IOError):
    >>>     import_module_from_path('does-not-exist.zip/')


if True:<EOL><INDENT>import importlib<EOL>module = importlib.import_module(modname)<EOL><DEDENT>else:<EOL><INDENT>if '<STR_LIT:.>' in modname:<EOL><INDENT>fromlist = modname.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>fromlist_ = list(map(str, fromlist))  <EOL>module = __import__(modname, {}, {}, fromlist_, <NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>module = __import__(modname, {}, {}, [], <NUM_LIT:0>)<EOL><DEDENT><DEDENT>return module<EOL>

Imports a module from its string name (__name__)

Args:
    modname (str):  module name

Returns:
    module: module

Example:
    >>> # test with modules that wont be imported in normal circumstances
    >>> # todo write a test where we gaurentee this
    >>> modname_list = [
    >>>     'pickletools',
    >>>     'lib2to3.fixes.fix_apply',
    >>> ]
    >>> #assert not any(m in sys.modules for m in modname_list)
    >>> modules = [import_module_from_name(modname) for modname in modname_list]
    >>> assert [m.__name__ for m in modules] == modname_list
    >>> assert all(m in sys.modules for m in modname_list)


import sysconfig<EOL>tags = []<EOL>if six.PY2:<EOL><INDENT>multiarch = sysconfig.get_config_var('<STR_LIT>')<EOL>if multiarch is not None:<EOL><INDENT>tags.append(multiarch)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>tags.append(sysconfig.get_config_var('<STR_LIT>'))<EOL>tags.append('<STR_LIT>')  <EOL><DEDENT>tags = [t for t in tags if t]<EOL>return tags<EOL>

Returns valid tags an extension module might have


import sysconfig<EOL>valid_exts = []<EOL>if six.PY2:<EOL><INDENT>base_ext = '<STR_LIT:.>' + sysconfig.get_config_var('<STR_LIT>').split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>base_ext = '<STR_LIT:.>' + sysconfig.get_config_var('<STR_LIT>').split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL><DEDENT>for tag in _extension_module_tags():<EOL><INDENT>valid_exts.append('<STR_LIT:.>' + tag + base_ext)<EOL><DEDENT>valid_exts.append(base_ext)<EOL>return tuple(valid_exts)<EOL>

Returns .so, .pyd, or .dylib depending on linux, win or mac.
On python3 return the previous with and without abi (e.g.
.cpython-35m-x86_64-linux-gnu) flags. On python2 returns with
and without multiarch.


def _syspath_modname_to_modpath(modname, sys_path=None, exclude=None):

def _isvalid(modpath, base):<EOL><INDENT>subdir = dirname(modpath)<EOL>while subdir and subdir != base:<EOL><INDENT>if not exists(join(subdir, '<STR_LIT>')):<EOL><INDENT>return False<EOL><DEDENT>subdir = dirname(subdir)<EOL><DEDENT>return True<EOL><DEDENT>_fname_we = modname.replace('<STR_LIT:.>', os.path.sep)<EOL>candidate_fnames = [<EOL>_fname_we + '<STR_LIT>',<EOL>]<EOL>candidate_fnames += [_fname_we + ext for ext in _platform_pylib_exts()]<EOL>if sys_path is None:<EOL><INDENT>sys_path = sys.path<EOL><DEDENT>candidate_dpaths = ['<STR_LIT:.>' if p == '<STR_LIT>' else p for p in sys_path]<EOL>if exclude:<EOL><INDENT>def normalize(p):<EOL><INDENT>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>return realpath(p).lower()<EOL><DEDENT>else:<EOL><INDENT>return realpath(p)<EOL><DEDENT><DEDENT>real_exclude = {normalize(p) for p in exclude}<EOL>candidate_dpaths = [p for p in candidate_dpaths<EOL>if normalize(p) not in real_exclude]<EOL><DEDENT>for dpath in candidate_dpaths:<EOL><INDENT>modpath = join(dpath, _fname_we)<EOL>if exists(modpath):<EOL><INDENT>if isfile(join(modpath, '<STR_LIT>')):<EOL><INDENT>if _isvalid(modpath, dpath):<EOL><INDENT>return modpath<EOL><DEDENT><DEDENT><DEDENT>for fname in candidate_fnames:<EOL><INDENT>modpath = join(dpath, fname)<EOL>if isfile(modpath):<EOL><INDENT>if _isvalid(modpath, dpath):<EOL><INDENT>return modpath<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

syspath version of modname_to_modpath

Args:
    modname (str): name of module to find
    sys_path (List[PathLike], default=None):
        if specified overrides `sys.path`
    exclude (List[PathLike], default=None):
        list of directory paths. if specified prevents these directories
        from being searched.

Notes:
    This is much slower than the pkgutil mechanisms.

CommandLine:
    python -m xdoctest.static_analysis _syspath_modname_to_modpath

Example:
    >>> print(_syspath_modname_to_modpath('xdoctest.static_analysis'))
    ...static_analysis.py
    >>> print(_syspath_modname_to_modpath('xdoctest'))
    ...xdoctest
    >>> print(_syspath_modname_to_modpath('_ctypes'))
    ..._ctypes...
    >>> assert _syspath_modname_to_modpath('xdoctest', sys_path=[]) is None
    >>> assert _syspath_modname_to_modpath('xdoctest.static_analysis', sys_path=[]) is None
    >>> assert _syspath_modname_to_modpath('_ctypes', sys_path=[]) is None
    >>> assert _syspath_modname_to_modpath('this', sys_path=[]) is None

Example:
    >>> # test what happens when the module is not visible in the path
    >>> modname = 'xdoctest.static_analysis'
    >>> modpath = _syspath_modname_to_modpath(modname)
    >>> exclude = [split_modpath(modpath)[0]]
    >>> found = _syspath_modname_to_modpath(modname, exclude=exclude)
    >>> # this only works if installed in dev mode, pypi fails
    >>> assert found is None, 'should not have found {}'.format(found)


def modname_to_modpath(modname, hide_init=True, hide_main=False, sys_path=None):

modpath = _syspath_modname_to_modpath(modname, sys_path)<EOL>if modpath is None:<EOL><INDENT>return None<EOL><DEDENT>modpath = normalize_modpath(modpath, hide_init=hide_init,<EOL>hide_main=hide_main)<EOL>return modpath<EOL>

Finds the path to a python module from its name.

Determines the path to a python module without directly import it

Converts the name of a module (__name__) to the path (__file__) where it is
located without importing the module. Returns None if the module does not
exist.

Args:
    modname (str): module filepath
    hide_init (bool): if False, __init__.py will be returned for packages
    hide_main (bool): if False, and hide_init is True, __main__.py will be
        returned for packages, if it exists.
    sys_path (list): if specified overrides `sys.path` (default None)

Returns:
    str: modpath - path to the module, or None if it doesn't exist

CommandLine:
    python -m xdoctest.static_analysis modname_to_modpath:0
    pytest  /home/joncrall/code/xdoctest/xdoctest/static_analysis.py::modname_to_modpath:0

Example:
    >>> modname = 'xdoctest.__main__'
    >>> modpath = modname_to_modpath(modname, hide_main=False)
    >>> assert modpath.endswith('__main__.py')
    >>> modname = 'xdoctest'
    >>> modpath = modname_to_modpath(modname, hide_init=False)
    >>> assert modpath.endswith('__init__.py')
    >>> modpath = basename(modname_to_modpath('_ctypes'))
    >>> assert 'ctypes' in modpath


def normalize_modpath(modpath, hide_init=True, hide_main=False):

if six.PY2:<EOL><INDENT>if modpath.endswith('<STR_LIT>'):<EOL><INDENT>modpath = modpath[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>if hide_init:<EOL><INDENT>if basename(modpath) == '<STR_LIT>':<EOL><INDENT>modpath = dirname(modpath)<EOL>hide_main = True<EOL><DEDENT><DEDENT>else:<EOL><INDENT>modpath_with_init = join(modpath, '<STR_LIT>')<EOL>if exists(modpath_with_init):<EOL><INDENT>modpath = modpath_with_init<EOL><DEDENT><DEDENT>if hide_main:<EOL><INDENT>if basename(modpath) == '<STR_LIT>':<EOL><INDENT>parallel_init = join(dirname(modpath), '<STR_LIT>')<EOL>if exists(parallel_init):<EOL><INDENT>modpath = dirname(modpath)<EOL><DEDENT><DEDENT><DEDENT>return modpath<EOL>

Normalizes __init__ and __main__ paths.

Notes:
    Adds __init__ if reasonable, but only removes __main__ by default

Args:
    hide_init (bool): if True, always return package modules
       as __init__.py files otherwise always return the dpath.
    hide_init (bool): if True, always strip away main files otherwise
       ignore __main__.py.

CommandLine:
    xdoctest -m xdoctest.static_analysis normalize_modpath

Example:
    >>> import xdoctest.static_analysis as static
    >>> modpath = static.__file__
    >>> assert static.normalize_modpath(modpath) == modpath.replace('.pyc', '.py')
    >>> dpath = dirname(modpath)
    >>> res0 = static.normalize_modpath(dpath, hide_init=0, hide_main=0)
    >>> res1 = static.normalize_modpath(dpath, hide_init=0, hide_main=1)
    >>> res2 = static.normalize_modpath(dpath, hide_init=1, hide_main=0)
    >>> res3 = static.normalize_modpath(dpath, hide_init=1, hide_main=1)
    >>> assert res0.endswith('__init__.py')
    >>> assert res1.endswith('__init__.py')
    >>> assert not res2.endswith('.py')
    >>> assert not res3.endswith('.py')


def modpath_to_modname(modpath, hide_init=True, hide_main=False, check=True,<EOL>relativeto=None):

if check and relativeto is None:<EOL><INDENT>if not exists(modpath):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT><DEDENT>modpath_ = abspath(expanduser(modpath))<EOL>modpath_ = normalize_modpath(modpath_, hide_init=hide_init,<EOL>hide_main=hide_main)<EOL>if relativeto:<EOL><INDENT>dpath = dirname(abspath(expanduser(relativeto)))<EOL>rel_modpath = relpath(modpath_, dpath)<EOL><DEDENT>else:<EOL><INDENT>dpath, rel_modpath = split_modpath(modpath_, check=check)<EOL><DEDENT>modname = splitext(rel_modpath)[<NUM_LIT:0>]<EOL>if '<STR_LIT:.>' in modname:<EOL><INDENT>modname, abi_tag = modname.split('<STR_LIT:.>')<EOL><DEDENT>modname = modname.replace('<STR_LIT:/>', '<STR_LIT:.>')<EOL>modname = modname.replace('<STR_LIT:\\>', '<STR_LIT:.>')<EOL>return modname<EOL>

Determines importable name from file path

Converts the path to a module (__file__) to the importable python name
(__name__) without importing the module.

The filename is converted to a module name, and parent directories are
recursively included until a directory without an __init__.py file is
encountered.

Args:
    modpath (str): module filepath
    hide_init (bool): removes the __init__ suffix (default True)
    hide_main (bool): removes the __main__ suffix (default False)
    check (bool): if False, does not raise an error if modpath is a dir
        and does not contain an __init__ file.
    relativeto (str, optional): if specified, all checks are ignored and
        this is considered the path to the root module.

Returns:
    str: modname

Raises:
    ValueError: if check is True and the path does not exist

CommandLine:
    xdoctest -m xdoctest.static_analysis modpath_to_modname

Example:
    >>> from xdoctest import static_analysis
    >>> modpath = static_analysis.__file__.replace('.pyc', '.py')
    >>> modpath = modpath.replace('.pyc', '.py')
    >>> modname = modpath_to_modname(modpath)
    >>> assert modname == 'xdoctest.static_analysis'

Example:
    >>> import xdoctest
    >>> assert modpath_to_modname(xdoctest.__file__.replace('.pyc', '.py')) == 'xdoctest'
    >>> assert modpath_to_modname(dirname(xdoctest.__file__.replace('.pyc', '.py'))) == 'xdoctest'

Example:
    >>> modpath = modname_to_modpath('_ctypes')
    >>> modname = modpath_to_modname(modpath)
    >>> assert modname == '_ctypes'


if six.PY2:<EOL><INDENT>if modpath.endswith('<STR_LIT>'):<EOL><INDENT>modpath = modpath[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>modpath_ = abspath(expanduser(modpath))<EOL>if check:<EOL><INDENT>if not exists(modpath_):<EOL><INDENT>if not exists(modpath):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT>if isdir(modpath_) and not exists(join(modpath, '<STR_LIT>')):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT><DEDENT>full_dpath, fname_ext = split(modpath_)<EOL>_relmod_parts = [fname_ext]<EOL>dpath = full_dpath<EOL>while exists(join(dpath, '<STR_LIT>')):<EOL><INDENT>dpath, dname = split(dpath)<EOL>_relmod_parts.append(dname)<EOL><DEDENT>relmod_parts = _relmod_parts[::-<NUM_LIT:1>]<EOL>rel_modpath = os.path.sep.join(relmod_parts)<EOL>return dpath, rel_modpath<EOL>

Splits the modpath into the dir that must be in PYTHONPATH for the module
to be imported and the modulepath relative to this directory.

Args:
    modpath (str): module filepath
    check (bool): if False, does not raise an error if modpath is a
        directory and does not contain an `__init__.py` file.

Returns:
    tuple: (directory, rel_modpath)

Raises:
    ValueError: if modpath does not exist or is not a package

Example:
    >>> from xdoctest import static_analysis
    >>> modpath = static_analysis.__file__.replace('.pyc', '.py')
    >>> modpath = abspath(modpath)
    >>> dpath, rel_modpath = split_modpath(modpath)
    >>> recon = join(dpath, rel_modpath)
    >>> assert recon == modpath
    >>> assert rel_modpath == join('xdoctest', 'static_analysis.py')


def symlink(real_path, link_path, overwrite=False, verbose=<NUM_LIT:0>):

path = normpath(real_path)<EOL>link = normpath(link_path)<EOL>if not os.path.isabs(path):<EOL><INDENT>if _can_symlink():<EOL><INDENT>path = os.path.relpath(path, os.path.dirname(link))<EOL><DEDENT>else:  <EOL><INDENT>path = os.path.abspath(path)<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path=path, link=link))<EOL><DEDENT>if islink(link):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>pointed = _readlink(link)<EOL>if pointed == path:<EOL><INDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>return link<EOL><DEDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>if not exists(link):<EOL><INDENT>print('<STR_LIT>'.format(pointed))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>'.format(pointed))<EOL><DEDENT><DEDENT>if overwrite:<EOL><INDENT>util_io.delete(link, verbose=verbose > <NUM_LIT:1>)<EOL><DEDENT><DEDENT>elif exists(link):<EOL><INDENT>if _win32_links is None:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>raise FileExistsError(<EOL>'<STR_LIT>'.format(path))<EOL><DEDENT>else:  <EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>if _win32_links is None:<EOL><INDENT>os.symlink(path, link)<EOL><DEDENT>else:  <EOL><INDENT>_win32_links._symlink(path, link, overwrite=overwrite, verbose=verbose)<EOL><DEDENT>return link<EOL>

Create a symbolic link.

This will work on linux or windows, however windows does have some corner
cases. For more details see notes in `ubelt._win32_links`.

Args:
    path (PathLike): path to real file or directory
    link_path (PathLike): path to desired location for symlink
    overwrite (bool): overwrite existing symlinks.
        This will not overwrite real files on systems with proper symlinks.
        However, on older versions of windows junctions are
        indistinguishable from real files, so we cannot make this
        guarantee.  (default = False)
    verbose (int):  verbosity level (default=0)

Returns:
    PathLike: link path

CommandLine:
    python -m ubelt.util_links symlink:0

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_cache_dir('ubelt', 'test_symlink0')
    >>> real_path = join(dpath, 'real_file.txt')
    >>> link_path = join(dpath, 'link_file.txt')
    >>> [ub.delete(p) for p in [real_path, link_path]]
    >>> ub.writeto(real_path, 'foo')
    >>> result = symlink(real_path, link_path)
    >>> assert ub.readfrom(result) == 'foo'
    >>> [ub.delete(p) for p in [real_path, link_path]]

Example:
    >>> import ubelt as ub
    >>> from os.path import dirname
    >>> dpath = ub.ensure_app_cache_dir('ubelt', 'test_symlink1')
    >>> ub.delete(dpath)
    >>> ub.ensuredir(dpath)
    >>> _dirstats(dpath)
    >>> real_dpath = ub.ensuredir((dpath, 'real_dpath'))
    >>> link_dpath = ub.augpath(real_dpath, base='link_dpath')
    >>> real_path = join(dpath, 'afile.txt')
    >>> link_path = join(dpath, 'afile.txt')
    >>> [ub.delete(p) for p in [real_path, link_path]]
    >>> ub.writeto(real_path, 'foo')
    >>> result = symlink(real_dpath, link_dpath)
    >>> assert ub.readfrom(link_path) == 'foo', 'read should be same'
    >>> ub.writeto(link_path, 'bar')
    >>> _dirstats(dpath)
    >>> assert ub.readfrom(link_path) == 'bar', 'very bad bar'
    >>> assert ub.readfrom(real_path) == 'bar', 'changing link did not change real'
    >>> ub.writeto(real_path, 'baz')
    >>> _dirstats(dpath)
    >>> assert ub.readfrom(real_path) == 'baz', 'very bad baz'
    >>> assert ub.readfrom(link_path) == 'baz', 'changing real did not change link'
    >>> ub.delete(link_dpath, verbose=1)
    >>> _dirstats(dpath)
    >>> assert not exists(link_dpath), 'link should not exist'
    >>> assert exists(real_path), 'real path should exist'
    >>> _dirstats(dpath)
    >>> ub.delete(dpath, verbose=1)
    >>> _dirstats(dpath)
    >>> assert not exists(real_path)


if _win32_links is not None:<EOL><INDENT>return _win32_links._win32_can_symlink(verbose)<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>

Return true if we have permission to create real symlinks.
This check always returns True on non-win32 systems.
If this check returns false, then we still may be able to use junctions.

CommandLine:
    python -m ubelt.util_platform _can_symlink

Example:
    >>> # Script
    >>> print(_can_symlink(verbose=1))


from ubelt import util_colors<EOL>if dpath is None:<EOL><INDENT>dpath = os.getcwd()<EOL><DEDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>'.format(dpath))<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>if not os.path.exists(dpath):<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>paths = sorted(os.listdir(dpath))<EOL>for path in paths:<EOL><INDENT>full_path = join(dpath, path)<EOL>E = os.path.exists(full_path)<EOL>L = os.path.islink(full_path)<EOL>F = os.path.isfile(full_path)<EOL>D = os.path.isdir(full_path)<EOL>J = util_platform.WIN32 and _win32_links._win32_is_junction(full_path)<EOL>ELFDJ = [E, L, F, D, J]<EOL>if   ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:1>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:1>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>'.format(dpath))<EOL>print('<STR_LIT>'.format(path))<EOL>raise AssertionError(str(ELFDJ) + str(path))<EOL><DEDENT>line = '<STR_LIT>'.format(**locals())<EOL>if os.path.islink(full_path):<EOL><INDENT>line += '<STR_LIT>' + os.readlink(full_path)<EOL><DEDENT>elif _win32_links is not None:<EOL><INDENT>if _win32_links._win32_is_junction(full_path):<EOL><INDENT>line += '<STR_LIT>' + _win32_links._win32_read_junction(full_path)<EOL><DEDENT><DEDENT>print(line)<EOL><DEDENT>

Testing helper for printing directory information
(mostly for investigating windows weirdness)

CommandLine:
    python -m ubelt.util_links _dirstats


return (self.redirect is not None and<EOL>hasattr(self.redirect, '<STR_LIT>') and self.redirect.isatty())<EOL>

Returns true of the redirect is a terminal.

Notes:
    Needed for IPython.embed to work properly when this class is used
    to override stdout / stderr.


if self.redirect is not None:<EOL><INDENT>return self.redirect.encoding<EOL><DEDENT>else:<EOL><INDENT>return super(TeeStringIO, self).encoding<EOL><DEDENT>

Gets the encoding of the `redirect` IO object

Doctest:
    >>> redirect = io.StringIO()
    >>> assert TeeStringIO(redirect).encoding is None
    >>> assert TeeStringIO(None).encoding is None
    >>> assert TeeStringIO(sys.stdout).encoding is sys.stdout.encoding
    >>> redirect = io.TextIOWrapper(io.StringIO())
    >>> assert TeeStringIO(redirect).encoding is redirect.encoding


if self.redirect is not None:<EOL><INDENT>self.redirect.write(msg)<EOL><DEDENT>if six.PY2:<EOL><INDENT>from xdoctest.utils.util_str import ensure_unicode<EOL>msg = ensure_unicode(msg)<EOL><DEDENT>super(TeeStringIO, self).write(msg)<EOL>


if self.redirect is not None:<EOL><INDENT>self.redirect.flush()<EOL><DEDENT>super(TeeStringIO, self).flush()<EOL>


self.cap_stdout.seek(self._pos)<EOL>text = self.cap_stdout.read()<EOL>self._pos = self.cap_stdout.tell()<EOL>self.parts.append(text)<EOL>self.text = text<EOL>


if self.enabled:<EOL><INDENT>self.started = False<EOL>sys.stdout = self.orig_stdout<EOL><DEDENT>

Doctest:
    >>> CaptureStdout(enabled=False).stop()
    >>> CaptureStdout(enabled=True).stop()


def augpath(path, suffix='<STR_LIT>', prefix='<STR_LIT>', ext=None, base=None, multidot=False):

<EOL>dpath, fname = split(path)<EOL>if multidot:<EOL><INDENT>parts = fname.split('<STR_LIT:.>')<EOL>fname_noext = '<STR_LIT:.>'.join(parts[:<NUM_LIT:1>])<EOL>orig_ext = '<STR_LIT:.>'.join(parts[<NUM_LIT:1>:])<EOL><DEDENT>else:<EOL><INDENT>fname_noext, orig_ext = splitext(fname)<EOL><DEDENT>ext = orig_ext if ext is None else ext<EOL>fname_noext = fname_noext if base is None else base<EOL>new_fname = '<STR_LIT>'.join((prefix, fname_noext, suffix, ext))<EOL>newpath = join(dpath, new_fname)<EOL>return newpath<EOL>

Augments a path with a new basename, extension, prefix and/or suffix.

A prefix is inserted before the basename. A suffix is inserted
between the basename and the extension. The basename and extension can be
replaced with a new one.

Args:
    path (PathLike): string representation of a path
    suffix (str): placed between the basename and extension
    prefix (str): placed in front of the basename
    ext (str): if specified, replaces the extension
    base (str): if specified, replaces the basename (without extension)
    multidot (bool): if False, everything after the last dot in the
        basename is the extension. If True, everything after the first dot
        in the basename is the extension (Defaults to False).

Returns:
    PathLike: augmented path

CommandLine:
    python -m ubelt.util_path augpath

Example:
    >>> import ubelt as ub
    >>> path = 'foo.bar'
    >>> suffix = '_suff'
    >>> prefix = 'pref_'
    >>> ext = '.baz'
    >>> newpath = ub.augpath(path, suffix, prefix, ext=ext, base='bar')
    >>> print('newpath = %s' % (newpath,))
    newpath = pref_bar_suff.baz

Example:
    >>> augpath('foo.bar')
    'foo.bar'
    >>> augpath('foo.bar', ext='.BAZ')
    'foo.BAZ'
    >>> augpath('foo.bar', suffix='_')
    'foo_.bar'
    >>> augpath('foo.bar', prefix='_')
    '_foo.bar'
    >>> augpath('foo.bar', base='baz')
    'baz.bar'
    >>> augpath('foo.tar.gz', ext='.zip', multidot=True)
    foo.zip
    >>> augpath('foo.tar.gz', ext='.zip', multidot=False)
    foo.tar.zip


if username is None:<EOL><INDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>userhome_dpath = os.environ['<STR_LIT>']<EOL><DEDENT>else:  <EOL><INDENT>if sys.platform.startswith('<STR_LIT:win32>'):<EOL><INDENT>if '<STR_LIT>' in os.environ:<EOL><INDENT>userhome_dpath = os.environ['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in os.environ:<EOL><INDENT>drive = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL>userhome_dpath = join(drive, os.environ['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>raise OSError("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>import pwd<EOL>userhome_dpath = pwd.getpwuid(os.getuid()).pw_dir<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>c_users = dirname(userhome())<EOL>userhome_dpath = join(c_users, username)<EOL>if not exists(userhome_dpath):<EOL><INDENT>raise KeyError('<STR_LIT>'.format(username))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>import pwd<EOL>try:<EOL><INDENT>pwent = pwd.getpwnam(username)<EOL><DEDENT>except KeyError:  <EOL><INDENT>raise KeyError('<STR_LIT>'.format(username))<EOL><DEDENT>userhome_dpath = pwent.pw_dir<EOL><DEDENT><DEDENT>return userhome_dpath<EOL>

Returns the user's home directory.
If `username` is None, this is the directory for the current user.

Args:
    username (str): name of a user on the system

Returns:
    PathLike: userhome_dpath: path to the home directory

Example:
    >>> import getpass
    >>> username = getpass.getuser()
    >>> assert userhome() == expanduser('~')
    >>> assert userhome(username) == expanduser('~')


path = normpath(path)<EOL>userhome_dpath = userhome()<EOL>if path.startswith(userhome_dpath):<EOL><INDENT>if len(path) == len(userhome_dpath):<EOL><INDENT>path = home<EOL><DEDENT>elif path[len(userhome_dpath)] == os.path.sep:<EOL><INDENT>path = home + path[len(userhome_dpath):]<EOL><DEDENT><DEDENT>return path<EOL>

Inverse of `os.path.expanduser`

Args:
    path (PathLike): path in system file structure
    home (str): symbol used to replace the home path. Defaults to '~', but
        you might want to use '$HOME' or '%USERPROFILE%' instead.

Returns:
    PathLike: path: shortened path replacing the home directory with a tilde

CommandLine:
    xdoctest -m ubelt.util_path compressuser

Example:
    >>> path = expanduser('~')
    >>> assert path != '~'
    >>> assert compressuser(path) == '~'
    >>> assert compressuser(path + '1') == path + '1'
    >>> assert compressuser(path + '/1') == join('~', '1')
    >>> assert compressuser(path + '/1', '$HOME') == join('$HOME', '1')


path = expanduser(path)<EOL>path = expandvars(path)<EOL>return path<EOL>

Wrapper around expanduser and expandvars.

Less aggressive than truepath. Only expands environs and tilde. Does not
change relative paths to absolute paths.

Args:
    path (PathLike): string representation of a path

Returns:
    PathLike : expanded path

Example:
    >>> import ubelt as ub
    >>> assert normpath(ub.expandpath('~/foo')) == join(ub.userhome(), 'foo')
    >>> assert ub.expandpath('foo') == 'foo'


path = expanduser(path)<EOL>path = expandvars(path)<EOL>if real:<EOL><INDENT>path = realpath(path)<EOL><DEDENT>else:<EOL><INDENT>path = abspath(path)<EOL><DEDENT>path = normpath(path)<EOL>return path<EOL>

Normalizes a string representation of a path and does shell-like expansion.

Args:
    path (PathLike): string representation of a path
    real (bool): if True, all symbolic links are followed. (default: False)

Returns:
    PathLike : normalized path

Note:
    This function is similar to the composition of expanduser, expandvars,
    normpath, and (realpath if `real` else abspath). However, on windows
    backslashes are then replaced with forward slashes to offer a
    consistent unix-like experience across platforms.

    On windows expanduser will expand environment variables formatted as
    %name%, whereas on unix, this will not occur.

CommandLine:
    python -m ubelt.util_path truepath

Example:
    >>> import ubelt as ub
    >>> assert ub.truepath('~/foo') == join(ub.userhome(), 'foo')
    >>> assert ub.truepath('~/foo') == ub.truepath('~/foo/bar/..')
    >>> assert ub.truepath('~/foo', real=True) == ub.truepath('~/foo')


def ensuredir(dpath, mode=<NUM_LIT>, verbose=None):

if verbose is None:  <EOL><INDENT>verbose = <NUM_LIT:0><EOL><DEDENT>if isinstance(dpath, (list, tuple)):  <EOL><INDENT>dpath = join(*dpath)<EOL><DEDENT>if not exists(dpath):<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>' % dpath)<EOL><DEDENT>if sys.version_info.major == <NUM_LIT:2>:  <EOL><INDENT>os.makedirs(normpath(dpath), mode=mode)<EOL><DEDENT>else:<EOL><INDENT>os.makedirs(normpath(dpath), mode=mode, exist_ok=True)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>' % dpath)<EOL><DEDENT><DEDENT>return dpath<EOL>

r"""
    Ensures that directory will exist. Creates new dir with sticky bits by
    default

    Args:
        dpath (PathLike): dir to ensure. Can also be a tuple to send to join
        mode (int): octal mode of directory (default 0o1777)
        verbose (int): verbosity (default 0)

    Returns:
        PathLike: path: the ensured directory

    Notes:
        This function is not thread-safe in Python2

    Example:
        >>> from ubelt.util_platform import *  # NOQA
        >>> import ubelt as ub
        >>> cache_dpath = ub.ensure_app_cache_dir('ubelt')
        >>> dpath = join(cache_dpath, 'ensuredir')
        >>> if exists(dpath):
        ...     os.rmdir(dpath)
        >>> assert not exists(dpath)
        >>> ub.ensuredir(dpath)
        >>> assert exists(dpath)
        >>> os.rmdir(dpath)


def download(url, fpath=None, hash_prefix=None, hasher='<STR_LIT>',<EOL>chunksize=<NUM_LIT>, verbose=<NUM_LIT:1>):

from progiter import ProgIter as Progress<EOL>from ubelt import util_platform<EOL>import shutil<EOL>import tempfile<EOL>import hashlib<EOL>if six.PY2:  <EOL><INDENT>from urllib2 import urlopen  <EOL><DEDENT>else:<EOL><INDENT>from urllib.request import urlopen  <EOL><DEDENT>if fpath is None:<EOL><INDENT>dpath = util_platform.ensure_app_cache_dir('<STR_LIT>')<EOL>fname = basename(url)<EOL>fpath = join(dpath, fname)<EOL><DEDENT>_dst_is_io_object = hasattr(fpath, '<STR_LIT>')<EOL>if verbose:<EOL><INDENT>if _dst_is_io_object:<EOL><INDENT>print('<STR_LIT>' % (url,))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' % (url, fpath))<EOL><DEDENT><DEDENT>urldata = urlopen(url)<EOL>meta = urldata.info()<EOL>try:<EOL><INDENT>if hasattr(meta, '<STR_LIT>'):  <EOL><INDENT>file_size = int(meta.getheaders("<STR_LIT>")[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>file_size = int(meta.get_all("<STR_LIT>")[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>except Exception:  <EOL><INDENT>file_size = None<EOL><DEDENT>if hash_prefix:<EOL><INDENT>if isinstance(hasher, six.string_types):<EOL><INDENT>if hasher == '<STR_LIT>':<EOL><INDENT>hasher = hashlib.sha1()<EOL><DEDENT>elif hasher == '<STR_LIT>':<EOL><INDENT>hasher = hashlib.sha512()<EOL><DEDENT>else:<EOL><INDENT>raise KeyError(hasher)<EOL><DEDENT><DEDENT><DEDENT>if _dst_is_io_object:<EOL><INDENT>_file_write = fpath.write<EOL><DEDENT>else:<EOL><INDENT>tmp = tempfile.NamedTemporaryFile(delete=False)<EOL>_file_write = tmp.write<EOL><DEDENT>_urldata_read = urldata.read<EOL>try:<EOL><INDENT>with Progress(total=file_size, disable=not verbose) as pbar:<EOL><INDENT>_pbar_update = pbar.update<EOL>def _critical_loop():<EOL><INDENT>buffer = '<STR_LIT:U+0020>'<EOL>if hash_prefix:<EOL><INDENT>_hasher_update = hasher.update<EOL>while buffer:<EOL><INDENT>buffer = _urldata_read(chunksize)<EOL>_file_write(buffer)<EOL>_hasher_update(buffer)<EOL>_pbar_update(len(buffer))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while buffer:<EOL><INDENT>buffer = _urldata_read(chunksize)<EOL>_file_write(buffer)<EOL>_pbar_update(len(buffer))<EOL><DEDENT><DEDENT><DEDENT>_critical_loop()<EOL><DEDENT>if not _dst_is_io_object:<EOL><INDENT>tmp.close()<EOL>shutil.move(tmp.name, fpath)<EOL><DEDENT>if hash_prefix:<EOL><INDENT>got = hasher.hexdigest()<EOL>if got[:len(hash_prefix)] != hash_prefix:<EOL><INDENT>print('<STR_LIT>'.format(hash_prefix))<EOL>print('<STR_LIT>'.format(got))<EOL>if _dst_is_io_object:<EOL><INDENT>raise RuntimeError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(hash_prefix, got))<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>fpath, hash_prefix, got))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if not _dst_is_io_object:  <EOL><INDENT>tmp.close()<EOL>if exists(tmp.name):<EOL><INDENT>os.remove(tmp.name)<EOL><DEDENT><DEDENT><DEDENT>return fpath<EOL>

downloads a url to a fpath.

Args:
    url (str):
        The url to download.

    fpath (PathLike | io.BytesIOtringIO):
        The path to download to. Defaults to basename of url and ubelt's
        application cache. If this is a io.BytesIO object then information
        is directly written to this object (note this prevents the use of
        temporary files).

    hash_prefix (None or str):
        If specified, download will retry / error if the file hash
        does not match this value. Defaults to None.

    hasher (str or Hasher):
        If hash_prefix is specified, this indicates the hashing
        algorithm to apply to the file. Defaults to sha512.

    chunksize (int):
        Download chunksize. Defaults to 2 ** 13.

    verbose (int):
        Verbosity level 0 or 1. Defaults to 1.

Returns:
    PathLike: fpath - file path string

Raises:
    URLError - if there is problem downloading the url
    RuntimeError - if the hash does not match the hash_prefix

Notes:
    Original code taken from pytorch in torch/utils/model_zoo.py and
    slightly modified.

References:
    http://blog.moleculea.com/2012/10/04/urlretrieve-progres-indicator/
    http://stackoverflow.com/questions/15644964/python-progress-bar-and-downloads
    http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py

CommandLine:
    python -m xdoctest ubelt.util_download download:1

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> from ubelt.util_download import *  # NOQA
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> fpath = download(url)
    >>> print(basename(fpath))
    rqwaDag.png

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> import ubelt as ub
    >>> import io
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> file = io.BytesIO()
    >>> fpath = download(url, file)
    >>> file.seek(0)
    >>> data = file.read()
    >>> assert ub.hash_data(data, hasher='sha1').startswith('f79ea24571')

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> fpath = download(url, hasher='sha1', hash_prefix='f79ea24571da6ddd2ba12e3d57b515249ecb8a35')
    Downloading url='http://i.imgur.com/rqwaDag.png' to fpath=...rqwaDag.png
    ...
    ...1233/1233... rate=... Hz, eta=..., total=..., wall=...

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> # test download from girder
    >>> import pytest
    >>> import ubelt as ub
    >>> url = 'https://data.kitware.com/api/v1/item/5b4039308d777f2e6225994c/download'
    >>> ub.download(url, hasher='sha512', hash_prefix='c98a46cb31205cf')
    >>> with pytest.raises(RuntimeError):
    >>>     ub.download(url, hasher='sha512', hash_prefix='BAD_HASH')


def grabdata(url, fpath=None, dpath=None, fname=None, redo=False,<EOL>verbose=<NUM_LIT:1>, appname=None, hash_prefix=None, hasher='<STR_LIT>',<EOL>**download_kw):

from ubelt import util_platform<EOL>if appname and dpath:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if fpath and (dpath or fname or appname):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>if fpath is None:<EOL><INDENT>if dpath is None:<EOL><INDENT>appname = appname or '<STR_LIT>'<EOL>dpath = util_platform.ensure_app_cache_dir(appname)<EOL><DEDENT>if fname is None:<EOL><INDENT>fname = basename(url)<EOL><DEDENT>fpath = join(dpath, fname)<EOL><DEDENT>needs_download = redo<EOL>if not exists(fpath):<EOL><INDENT>needs_download = True<EOL><DEDENT>if hash_prefix:<EOL><INDENT>stamp_fpath, needs_download = _check_hash_stamp(<EOL>fpath, hash_prefix, hasher, verbose, needs_download)<EOL><DEDENT>if needs_download:<EOL><INDENT>fpath = download(url, fpath, verbose=verbose,<EOL>hash_prefix=hash_prefix, hasher=hasher,<EOL>**download_kw)<EOL>if hash_prefix:<EOL><INDENT>with open(stamp_fpath, '<STR_LIT:w>') as file:<EOL><INDENT>file.write(hash_prefix)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if verbose >= <NUM_LIT:2>:<EOL><INDENT>print('<STR_LIT>' % fpath)<EOL><DEDENT><DEDENT>return fpath<EOL>

Downloads a file, caches it, and returns its local path.

Args:
    url (str): url to the file to download

    fpath (PathLike): The full path to download the file to. If
        unspecified, the arguments `dpath` and `fname` are used to
        determine this.

    dpath (PathLike): where to download the file. If unspecified `appname`
        is used to determine this. Mutually exclusive with fpath.

    fname (str): What to name the downloaded file. Defaults to the url
        basename. Mutually exclusive with fpath.

    redo (bool): if True forces redownload of the file (default = False)

    verbose (bool):  verbosity flag (default = True)

    appname (str): set dpath to `ub.get_app_cache_dir(appname)`.
        Mutually exclusive with dpath and fpath.

    hash_prefix (None or str):
        If specified, grabdata verifies that this matches the hash of the
        file, and then saves the hash in a adjacent file to certify that
        the download was successful. Defaults to None.

    hasher (str or Hasher):
        If hash_prefix is specified, this indicates the hashing
        algorithm to apply to the file. Defaults to sha512.

    **download_kw: additional kwargs to pass to ub.download

Returns:
    PathLike: fpath - file path string

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> import ubelt as ub
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> fpath = ub.grabdata(url, fname='mario.png')
    >>> result = basename(fpath)
    >>> print(result)
    mario.png

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> import ubelt as ub
    >>> fname = 'foo.bar'
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> prefix1 = '944389a39dfb8fa9'
    >>> fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
    >>> stamp_fpath = fpath + '.hash'
    >>> assert open(stamp_fpath, 'r').read() == prefix1
    >>> # Check that the download doesn't happen again
    >>> fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
    >>> # todo: check file timestamps have not changed
    >>> #
    >>> # Check redo works with hash
    >>> fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1, redo=True)
    >>> # todo: check file timestamps have changed
    >>> #
    >>> # Check that a redownload occurs when the stamp is changed
    >>> open(stamp_fpath, 'w').write('corrupt-stamp')
    >>> fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
    >>> assert open(stamp_fpath, 'r').read() == prefix1
    >>> #
    >>> # Check that a redownload occurs when the stamp is removed
    >>> ub.delete(stamp_fpath)
    >>> open(fpath, 'w').write('corrupt-data')
    >>> assert not ub.hash_file(fpath, base='hex', hasher='sha512').startswith(prefix1)
    >>> fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
    >>> assert ub.hash_file(fpath, base='hex', hasher='sha512').startswith(prefix1)
    >>> #
    >>> # Check that requesting new data causes redownload
    >>> url2 = 'https://data.kitware.com/api/v1/item/5b4039308d777f2e6225994c/download'
    >>> prefix2 = 'c98a46cb31205cf'
    >>> fpath = ub.grabdata(url2, fname=fname, hash_prefix=prefix2)
    >>> assert open(stamp_fpath, 'r').read() == prefix2


line = stream.readline()<EOL>while line != '<STR_LIT>':<EOL><INDENT>yield line<EOL>line = stream.readline()<EOL><DEDENT>

Iterates over lines in a TextIO stream until an EOF is encountered.
This is the iterator version of stream.readlines()


def _proc_async_iter_stream(proc, stream, buffersize=<NUM_LIT:1>):

from six.moves import queue<EOL>from threading import Thread<EOL>def enqueue_output(proc, stream, stream_queue):<EOL><INDENT>while proc.poll() is None:<EOL><INDENT>line = stream.readline()<EOL>stream_queue.put(line)<EOL><DEDENT>for line in _textio_iterlines(stream):<EOL><INDENT>stream_queue.put(line)<EOL><DEDENT>stream_queue.put(None)  <EOL><DEDENT>stream_queue = queue.Queue(maxsize=buffersize)<EOL>_thread = Thread(target=enqueue_output, args=(proc, stream, stream_queue))<EOL>_thread.daemon = True  <EOL>_thread.start()<EOL>return stream_queue<EOL>

Reads output from a process in a separate thread


from six.moves import queue<EOL>stdout_queue = _proc_async_iter_stream(proc, proc.stdout)<EOL>stderr_queue = _proc_async_iter_stream(proc, proc.stderr)<EOL>stdout_live = True<EOL>stderr_live = True<EOL>while stdout_live or stderr_live:<EOL><INDENT>if stdout_live:  <EOL><INDENT>try:<EOL><INDENT>oline = stdout_queue.get_nowait()<EOL>stdout_live = oline is not None<EOL><DEDENT>except queue.Empty:<EOL><INDENT>oline = None<EOL><DEDENT><DEDENT>if stderr_live:<EOL><INDENT>try:<EOL><INDENT>eline = stderr_queue.get_nowait()<EOL>stderr_live = eline is not None<EOL><DEDENT>except queue.Empty:<EOL><INDENT>eline = None<EOL><DEDENT><DEDENT>if oline is not None or eline is not None:<EOL><INDENT>yield oline, eline<EOL><DEDENT><DEDENT>

Iterates over output from a process line by line

Note:
    WARNING. Current implementation might have bugs with other threads.
    This behavior was seen when using earlier versions of tqdm. I'm not
    sure if this was our bug or tqdm's. Newer versions of tqdm fix this,
    but I cannot guarantee that there isn't an issue on our end.

Yields:
    Tuple[str, str]: oline, eline: stdout and stderr line

References:
    https://stackoverflow.com/questions/375427/non-blocking-read-subproc


from six.moves import zip_longest<EOL>while proc.poll() is None:<EOL><INDENT>reads = [proc.stdout.fileno(), proc.stderr.fileno()]<EOL>ret = select.select(reads, [], [])<EOL>oline = eline = None<EOL>for fd in ret[<NUM_LIT:0>]:<EOL><INDENT>if fd == proc.stdout.fileno():<EOL><INDENT>oline = proc.stdout.readline()<EOL><DEDENT>if fd == proc.stderr.fileno():<EOL><INDENT>eline = proc.stderr.readline()<EOL><DEDENT><DEDENT>yield oline, eline<EOL><DEDENT>oline_iter = _textio_iterlines(proc.stdout)<EOL>eline_iter = _textio_iterlines(proc.stderr)<EOL>for oline, eline in zip_longest(oline_iter, eline_iter):<EOL><INDENT>yield oline, eline<EOL><DEDENT>

Iterates over output from a process line by line

UNIX only. Use `_proc_iteroutput_thread` instead for a cross platform
solution based on threads.

Yields:
    Tuple[str, str]: oline, eline: stdout and stderr line


def _tee_output(make_proc, stdout=None, stderr=None, backend='<STR_LIT>'):

logged_out = []<EOL>logged_err = []<EOL>if backend == '<STR_LIT>':<EOL><INDENT>backend = '<STR_LIT>'<EOL><DEDENT>if backend == '<STR_LIT>':<EOL><INDENT>if not POSIX:  <EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>_proc_iteroutput = _proc_iteroutput_select<EOL><DEDENT>elif backend == '<STR_LIT>':<EOL><INDENT>_proc_iteroutput = _proc_iteroutput_thread<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>proc = make_proc()<EOL>for oline, eline in _proc_iteroutput(proc):<EOL><INDENT>if oline:<EOL><INDENT>if stdout:  <EOL><INDENT>stdout.write(oline)<EOL>stdout.flush()<EOL><DEDENT>logged_out.append(oline)<EOL><DEDENT>if eline:<EOL><INDENT>if stderr:  <EOL><INDENT>stderr.write(eline)<EOL>stderr.flush()<EOL><DEDENT>logged_err.append(eline)<EOL><DEDENT><DEDENT>return proc, logged_out, logged_err<EOL>

Simultaneously reports and captures stdout and stderr from a process

subprocess must be created using (stdout=subprocess.PIPE,
stderr=subprocess.PIPE)


def cmd(command, shell=False, detach=False, verbose=<NUM_LIT:0>, tee=None, cwd=None,<EOL>env=None, tee_backend='<STR_LIT>', verbout=None, **kwargs):

if kwargs:  <EOL><INDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>warnings.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', DeprecationWarning)<EOL>tee = kwargs.pop('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>warnings.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', DeprecationWarning)<EOL>detach = kwargs.pop('<STR_LIT>')<EOL><DEDENT>if kwargs:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(list(kwargs.keys)))<EOL><DEDENT><DEDENT>if isinstance(command, six.string_types):<EOL><INDENT>command_text = command<EOL>command_tup = None<EOL><DEDENT>else:<EOL><INDENT>import pipes<EOL>command_tup = command<EOL>command_text = '<STR_LIT:U+0020>'.join(list(map(pipes.quote, command_tup)))<EOL><DEDENT>if shell or sys.platform.startswith('<STR_LIT:win32>'):<EOL><INDENT>args = command_text<EOL><DEDENT>else:<EOL><INDENT>if command_tup is None:<EOL><INDENT>import shlex<EOL>command_tup = shlex.split(command_text)<EOL><DEDENT>args = command_tup<EOL><DEDENT>if tee is None:<EOL><INDENT>tee = verbose > <NUM_LIT:0><EOL><DEDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>import os<EOL>import platform<EOL>import getpass<EOL>from ubelt import util_path<EOL>if verbose > <NUM_LIT:2>:<EOL><INDENT>try:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>except Exception:  <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT>cwd_ = os.getcwd() if cwd is None else cwd<EOL>compname = platform.node()<EOL>username = getpass.getuser()<EOL>cwd_ = util_path.compressuser(cwd_)<EOL>ps1 = '<STR_LIT>'.format(username, compname, cwd_)<EOL>print(ps1 + command_text)<EOL><DEDENT>def make_proc():<EOL><INDENT>import subprocess<EOL>proc = subprocess.Popen(args, stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE, shell=shell,<EOL>universal_newlines=True, cwd=cwd, env=env)<EOL>return proc<EOL><DEDENT>if detach:<EOL><INDENT>info = {'<STR_LIT>': make_proc(), '<STR_LIT>': command_text}<EOL>if verbose > <NUM_LIT:0>:  <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if tee:<EOL><INDENT>stdout, stderr = sys.stdout, sys.stderr<EOL>proc, logged_out, logged_err = _tee_output(make_proc, stdout, stderr,<EOL>backend=tee_backend)<EOL>try:<EOL><INDENT>out = '<STR_LIT>'.join(logged_out)<EOL><DEDENT>except UnicodeDecodeError:  <EOL><INDENT>out = '<STR_LIT:\n>'.join(_.decode('<STR_LIT:utf-8>') for _ in logged_out)<EOL><DEDENT>try:<EOL><INDENT>err = '<STR_LIT>'.join(logged_err)<EOL><DEDENT>except UnicodeDecodeError:  <EOL><INDENT>err = '<STR_LIT:\n>'.join(_.decode('<STR_LIT:utf-8>') for _ in logged_err)<EOL><DEDENT>(out_, err_) = proc.communicate()<EOL><DEDENT>else:<EOL><INDENT>proc = make_proc()<EOL>(out, err) = proc.communicate()<EOL><DEDENT>ret = proc.wait()<EOL>info = {<EOL>'<STR_LIT>': out,<EOL>'<STR_LIT>': err,<EOL>'<STR_LIT>': ret,<EOL>'<STR_LIT>': proc,<EOL>'<STR_LIT>': cwd,<EOL>'<STR_LIT>': command_text<EOL>}<EOL>if verbose > <NUM_LIT:2>:<EOL><INDENT>try:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>except Exception:  <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>return info<EOL>

Executes a command in a subprocess.

The advantage of this wrapper around subprocess is that
(1) you control if the subprocess prints to stdout,
(2) the text written to stdout and stderr is returned for parsing,
(3) cross platform behavior that lets you specify the command as a string
or tuple regardless of whether or not shell=True.
(4) ability to detach, return the process object and allow the process to
run in the background (eventually we may return a Future object instead).

Args:
    command (str or Sequence): bash-like command string or tuple of
        executable and args

    shell (bool): if True, process is run in shell, defaults to False.

    detach (bool): if True, process is detached and run in background,
        defaults to False.

    verbose (int): verbosity mode. Can be 0, 1, 2, or 3. Defaults to 0.

    tee (bool, optional): if True, simultaneously writes to stdout while
        capturing output from the command. If not specified, defaults to
        True if verbose > 0.  If detech is True, then this argument is
        ignored.

    cwd (PathLike, optional): path to run command

    env (str, optional): environment passed to Popen

    tee_backend (str, optional): backend for tee output.
        Valid choices are: "auto", "select" (POSIX only), and "thread".

    **kwargs: only used to support deprecated arguments

Returns:
    dict: info - information about command status.
        if detach is False `info` contains captured standard out,
        standard error, and the return code
        if detach is False `info` contains a reference to the process.

Notes:
    Inputs can either be text or tuple based. On UNIX we ensure conversion
    to text if shell=True, and to tuple if shell=False. On windows, the
    input is always text based.  See [3] for a potential cross-platform
    shlex solution for windows.

CommandLine:
    python -m ubelt.util_cmd cmd
    python -c "import ubelt as ub; ub.cmd('ping localhost -c 2', verbose=2)"

References:
    [1] https://stackoverflow.com/questions/11495783/redirect-subprocess-stderr-to-stdout
    [2] https://stackoverflow.com/questions/7729336/how-can-i-print-and-display-subprocess-stdout-and-stderr-output-without-distorti
    [3] https://stackoverflow.com/questions/33560364/python-windows-parsing-command-lines-with-shlex

Example:
    >>> info = cmd(('echo', 'simple cmdline interface'), verbose=1)
    simple cmdline interface
    >>> assert info['ret'] == 0
    >>> assert info['out'].strip() == 'simple cmdline interface'
    >>> assert info['err'].strip() == ''

Doctest:
    >>> info = cmd('echo str noshell', verbose=0)
    >>> assert info['out'].strip() == 'str noshell'

Doctest:
    >>> # windows echo will output extra single quotes
    >>> info = cmd(('echo', 'tuple noshell'), verbose=0)
    >>> assert info['out'].strip().strip("'") == 'tuple noshell'

Doctest:
    >>> # Note this command is formatted to work on win32 and unix
    >>> info = cmd('echo str&&echo shell', verbose=0, shell=True)
    >>> assert info['out'].strip() == 'str' + chr(10) + 'shell'

Doctest:
    >>> info = cmd(('echo', 'tuple shell'), verbose=0, shell=True)
    >>> assert info['out'].strip().strip("'") == 'tuple shell'

Doctest:
    >>> import ubelt as ub
    >>> from os.path import join, exists
    >>> fpath1 = join(ub.get_app_cache_dir('ubelt'), 'cmdout1.txt')
    >>> fpath2 = join(ub.get_app_cache_dir('ubelt'), 'cmdout2.txt')
    >>> ub.delete(fpath1)
    >>> ub.delete(fpath2)
    >>> info1 = ub.cmd(('touch', fpath1), detach=True)
    >>> info2 = ub.cmd('echo writing2 > ' + fpath2, shell=True, detach=True)
    >>> while not exists(fpath1):
    ...     pass
    >>> while not exists(fpath2):
    ...     pass
    >>> assert ub.readfrom(fpath1) == ''
    >>> assert ub.readfrom(fpath2).strip() == 'writing2'
    >>> info1['proc'].wait()
    >>> info2['proc'].wait()


if method == '<STR_LIT>':<EOL><INDENT>tz_hour = time.timezone // <NUM_LIT><EOL>utc_offset = str(tz_hour) if tz_hour < <NUM_LIT:0> else '<STR_LIT:+>' + str(tz_hour)<EOL>stamp = time.strftime('<STR_LIT>') + utc_offset<EOL>return stamp<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

make an iso8601 timestamp

Args:
    method (str): type of timestamp

Example:
    >>> stamp = timestamp()
    >>> print('stamp = {!r}'.format(stamp))
    stamp = ...-...-...T...


try:<EOL><INDENT>iter(obj)<EOL><DEDENT>except Exception:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return strok or not isinstance(obj, six.string_types)<EOL><DEDENT>

Checks if the input implements the iterator interface. An exception is made
for strings, which return False unless `strok` is True

Args:
    obj (object): a scalar or iterable input

    strok (bool): if True allow strings to be interpreted as iterable

Returns:
    bool: True if the input is iterable

Example:
    >>> obj_list = [3, [3], '3', (3,), [3, 4, 5], {}]
    >>> result = [iterable(obj) for obj in obj_list]
    >>> assert result == [False, True, False, True, True, True]
    >>> result = [iterable(obj, strok=True) for obj in obj_list]
    >>> assert result == [False, True, True, True, True, True]


return (items[index] for index in indices)<EOL>

Selects a subset of a list based on a list of indices.
This is similar to np.take, but pure python.

Args:
    items (Sequence): an indexable object to select items from

    indices (Iterable): sequence of indexing objects

Returns:
    Iterable or scalar: subset of the list

SeeAlso:
    ub.dict_subset

Example:
    >>> import ubelt as ub
    >>> items = [0, 1, 2, 3]
    >>> indices = [2, 0]
    >>> list(ub.take(items, indices))
    [2, 0]


return it.compress(items, flags)<EOL>

Selects items where the corresponding value in flags is True
This is similar to np.compress and it.compress

Args:
    items (Iterable): a sequence to select items from

    flags (Iterable): corresponding sequence of bools

Returns:
    Iterable: a subset of masked items

Example:
    >>> import ubelt as ub
    >>> items = [1, 2, 3, 4, 5]
    >>> flags = [False, True, True, False, True]
    >>> list(ub.compress(items, flags))
    [2, 3, 5]


return it.chain.from_iterable(nested_list)<EOL>

Transforms a nested iterable into a flat iterable.

This is simply an alias for `itertools.chain.from_iterable`

Args:
    nested_list (Iterable[Iterable]): list of lists

Returns:
    Iterable: flattened items

Example:
    >>> import ubelt as ub
    >>> nested_list = [['a', 'b'], ['c', 'd']]
    >>> list(ub.flatten(nested_list))
    ['a', 'b', 'c', 'd']


seen = set()<EOL>if key is None:<EOL><INDENT>for item in items:<EOL><INDENT>if item not in seen:<EOL><INDENT>seen.add(item)<EOL>yield item<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for item in items:<EOL><INDENT>norm = key(item)<EOL>if norm not in seen:<EOL><INDENT>seen.add(norm)<EOL>yield item<EOL><DEDENT><DEDENT><DEDENT>

Generates unique items in the order they appear.

Args:
    items (Iterable): list of items

    key (Callable, optional): custom normalization function.
        If specified returns items where `key(item)` is unique.

Yields:
    object: a unique item from the input sequence

CommandLine:
    python -m utool.util_list --exec-unique_ordered

Example:
    >>> import ubelt as ub
    >>> items = [4, 6, 6, 0, 6, 1, 0, 2, 2, 1]
    >>> unique_items = list(ub.unique(items))
    >>> assert unique_items == [4, 6, 0, 1, 2]

Example:
    >>> import ubelt as ub
    >>> items = ['A', 'a', 'b', 'B', 'C', 'c', 'D', 'e', 'D', 'E']
    >>> unique_items = list(ub.unique(items, key=six.text_type.lower))
    >>> assert unique_items == ['A', 'b', 'C', 'D', 'e']
    >>> unique_items = list(ub.unique(items))
    >>> assert unique_items == ['A', 'a', 'b', 'B', 'C', 'c', 'D', 'e', 'E']


<EOL>if key is None:<EOL><INDENT>return unique(range(len(items)), key=lambda i: items[i])<EOL><DEDENT>else:<EOL><INDENT>return unique(range(len(items)), key=lambda i: key(items[i]))<EOL><DEDENT>

Returns indices corresponding to the first instance of each unique item.

Args:
    items (Sequence): indexable collection of items

    key (Callable, optional): custom normalization function.
        If specified returns items where `key(item)` is unique.

Yields:
    int : indices of the unique items

Example:
    >>> items = [0, 2, 5, 1, 1, 0, 2, 4]
    >>> indices = list(argunique(items))
    >>> assert indices == [0, 1, 2, 3, 7]
    >>> indices = list(argunique(items, key=lambda x: x % 2 == 0))
    >>> assert indices == [0, 2]