@self.register(uuid.UUID)<EOL>def _hash_uuid(data):<EOL><INDENT>hashable = data.bytes<EOL>prefix = b'<STR_LIT>'<EOL>return prefix, hashable<EOL><DEDENT>@self.register(OrderedDict)<EOL>def _hash_ordered_dict(data):<EOL><INDENT>"""<STR_LIT>"""<EOL>hashable = b'<STR_LIT>'.join(_hashable_sequence(list(data.items())))<EOL>prefix = b'<STR_LIT>'<EOL>return prefix, hashable<EOL><DEDENT>

Register hashing extensions for a selection of classes included in
python stdlib.

Example:
    >>> data = uuid.UUID('7e9d206b-dc02-4240-8bdb-fffe858121d0')
    >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
    cryarepd
    >>> data = OrderedDict([('a', 1), ('b', 2), ('c', [1, 2, 3]),
    >>>                     (4, OrderedDict())])
    >>> print(hash_data(data, base='abc', hasher='sha512', types=True)[0:8])
    qjspicvv

    gpxtclct


if base is _ALPHABET_16:<EOL><INDENT>return hexstr<EOL><DEDENT>baselen = len(base)<EOL>x = int(hexstr, <NUM_LIT:16>)  <EOL>if x == <NUM_LIT:0>:<EOL><INDENT>return '<STR_LIT:0>'<EOL><DEDENT>sign = <NUM_LIT:1> if x > <NUM_LIT:0> else -<NUM_LIT:1><EOL>x *= sign<EOL>digits = []<EOL>while x:<EOL><INDENT>digits.append(base[x % baselen])<EOL>x //= baselen<EOL><DEDENT>if sign < <NUM_LIT:0>:<EOL><INDENT>digits.append('<STR_LIT:->')<EOL><DEDENT>digits.reverse()<EOL>newbase_str = '<STR_LIT>'.join(digits)<EOL>return newbase_str<EOL>

r"""
    Packs a long hexstr into a shorter length string with a larger base.

    Args:
        hexstr (str): string of hexidecimal symbols to convert
        base (list): symbols of the conversion base

    Example:
        >>> print(_convert_hexstr_base('ffffffff', _ALPHABET_26))
        nxmrlxv
        >>> print(_convert_hexstr_base('0', _ALPHABET_26))
        0
        >>> print(_convert_hexstr_base('-ffffffff', _ALPHABET_26))
        -nxmrlxv
        >>> print(_convert_hexstr_base('aafffff1', _ALPHABET_16))
        aafffff1

    Sympy:
        >>> import sympy as sy
        >>> # Determine the length savings with lossless conversion
        >>> consts = dict(hexbase=16, hexlen=256, baselen=27)
        >>> symbols = sy.symbols('hexbase, hexlen, baselen, newlen')
        >>> haexbase, hexlen, baselen, newlen = symbols
        >>> eqn = sy.Eq(16 ** hexlen,  baselen ** newlen)
        >>> newlen_ans = sy.solve(eqn, newlen)[0].subs(consts).evalf()
        >>> print('newlen_ans = %r' % (newlen_ans,))
        >>> # for a 26 char base we can get 216
        >>> print('Required length for lossless conversion len2 = %r' % (len2,))
        >>> def info(base, len):
        ...     bits = base ** len
        ...     print('base = %r' % (base,))
        ...     print('len = %r' % (len,))
        ...     print('bits = %r' % (bits,))
        >>> info(16, 256)
        >>> info(27, 16)
        >>> info(27, 64)
        >>> info(27, 216)


def hash_file(fpath, blocksize=<NUM_LIT>, stride=<NUM_LIT:1>, hasher=NoParam,<EOL>hashlen=NoParam, base=NoParam):

base = _rectify_base(base)<EOL>hashlen = _rectify_hashlen(hashlen)<EOL>hasher = _rectify_hasher(hasher)()<EOL>with open(fpath, '<STR_LIT:rb>') as file:<EOL><INDENT>buf = file.read(blocksize)<EOL>if stride > <NUM_LIT:1>:<EOL><INDENT>while len(buf) > <NUM_LIT:0>:<EOL><INDENT>hasher.update(buf)<EOL>file.seek(blocksize * (stride - <NUM_LIT:1>), <NUM_LIT:1>)<EOL>buf = file.read(blocksize)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while len(buf) > <NUM_LIT:0>:<EOL><INDENT>hasher.update(buf)<EOL>buf = file.read(blocksize)<EOL><DEDENT><DEDENT><DEDENT>text = _digest_hasher(hasher, hashlen, base)<EOL>return text<EOL>

Hashes the data in a file on disk.

Args:
    fpath (PathLike):  file path string

    blocksize (int): 2 ** 16. Affects speed of reading file

    stride (int): strides > 1 skip data to hash, useful for faster
                  hashing, but less accurate, also makes hash dependant on
                  blocksize.

    hasher (HASH): hash algorithm from hashlib, defaults to `sha512`.

    hashlen (int): maximum number of symbols in the returned hash. If
        not specified, all are returned.

    base (list, str): list of symbols or shorthand key. Valid keys are
        'abc', 'hex', and 'dec'. Defaults to 'hex'.

Notes:
    For better hashes keep stride = 1
    For faster hashes set stride > 1
    blocksize matters when stride > 1

References:
    http://stackoverflow.com/questions/3431825/md5-checksum-of-a-file
    http://stackoverflow.com/questions/5001893/when-to-use-sha-1-vs-sha-2

Example:
    >>> import ubelt as ub
    >>> from os.path import join
    >>> fpath = join(ub.ensure_app_cache_dir('ubelt'), 'tmp.txt')
    >>> ub.writeto(fpath, 'foobar')
    >>> print(ub.hash_file(fpath, hasher='sha1', base='hex'))
    8843d7f92416211de9ebb963ff4ce28125932878

Example:
    >>> import ubelt as ub
    >>> from os.path import join
    >>> fpath = ub.touch(join(ub.ensure_app_cache_dir('ubelt'), 'empty_file'))
    >>> # Test that the output is the same as sha1sum
    >>> if ub.find_exe('sha1sum'):
    >>>     want = ub.cmd(['sha1sum', fpath], verbose=2)['out'].split(' ')[0]
    >>>     got = ub.hash_file(fpath, hasher='sha1')
    >>>     print('want = {!r}'.format(want))
    >>>     print('got = {!r}'.format(got))
    >>>     assert want.endswith(got)
    >>> # Do the same for sha512 sum and md5sum
    >>> if ub.find_exe('sha512sum'):
    >>>     want = ub.cmd(['sha512sum', fpath], verbose=2)['out'].split(' ')[0]
    >>>     got = ub.hash_file(fpath, hasher='sha512')
    >>>     print('want = {!r}'.format(want))
    >>>     print('got = {!r}'.format(got))
    >>>     assert want.endswith(got)
    >>> if ub.find_exe('md5sum'):
    >>>     want = ub.cmd(['md5sum', fpath], verbose=2)['out'].split(' ')[0]
    >>>     got = ub.hash_file(fpath, hasher='md5')
    >>>     print('want = {!r}'.format(want))
    >>>     print('got = {!r}'.format(got))
    >>>     assert want.endswith(got)


<EOL>if data is None:<EOL><INDENT>hashable = b'<STR_LIT>'<EOL>prefix = b'<STR_LIT>'<EOL><DEDENT>elif isinstance(data, six.binary_type):<EOL><INDENT>hashable = data<EOL>prefix = b'<STR_LIT>'<EOL><DEDENT>elif isinstance(data, six.text_type):<EOL><INDENT>hashable = data.encode('<STR_LIT:utf-8>')<EOL>prefix = b'<STR_LIT>'<EOL><DEDENT>elif isinstance(data, _intlike):<EOL><INDENT>hashable = _int_to_bytes(data)<EOL>prefix = b'<STR_LIT>'<EOL><DEDENT>elif isinstance(data, float):<EOL><INDENT>a, b = float(data).as_integer_ratio()<EOL>hashable = _int_to_bytes(a) + b'<STR_LIT:/>' +  _int_to_bytes(b)<EOL>prefix = b'<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>hash_func = _HASHABLE_EXTENSIONS.lookup(data)<EOL>prefix, hashable = hash_func(data)<EOL><DEDENT>if types:<EOL><INDENT>return prefix, hashable<EOL><DEDENT>else:<EOL><INDENT>return b'<STR_LIT>', hashable<EOL><DEDENT>

r"""
    Converts `data` into a hashable byte representation if an appropriate
    hashing function is known.

    Args:
        data (object): ordered data with structure
        types (bool): include type prefixes in the hash

    Returns:
        tuple(bytes, bytes): prefix, hashable:
            a prefix hinting the original data type and the byte representation
            of `data`.

    Raises:
        TypeError : if data has no registered hash methods

    Example:
        >>> assert _convert_to_hashable(None) == (b'NULL', b'NONE')
        >>> assert _convert_to_hashable('string') == (b'TXT', b'string')
        >>> assert _convert_to_hashable(1) == (b'INT', b'\x01')
        >>> assert _convert_to_hashable(1.0) == (b'FLT', b'\x01/\x01')
        >>> assert _convert_to_hashable(_intlike[-1](1)) == (b'INT', b'\x01')


import textwrap  <EOL>return textwrap.dedent(block_str).strip('<STR_LIT:\n>')<EOL>

Wraps multiline string blocks and returns unindented code.
Useful for templated code defined in indented parts of code.

Args:
    block_str (str): typically in the form of a multiline string

Returns:
    str: the unindented string

CommandLine:
    python -m ubelt.util_str codeblock

Example:
    >>> from ubelt.util_str import *  # NOQA
    >>> # Simulate an indented part of code
    >>> if True:
    >>>     # notice the indentation on this will be normal
    >>>     codeblock_version = codeblock(
    ...             '''
    ...             def foo():
    ...                 return 'bar'
    ...             '''
    ...         )
    >>>     # notice the indentation and newlines on this will be odd
    >>>     normal_version = ('''
    ...         def foo():
    ...             return 'bar'
    ...     ''')
    >>> assert normal_version != codeblock_version
    >>> print('Without codeblock')
    >>> print(normal_version)
    >>> print('With codeblock')
    >>> print(codeblock_version)


def _join_itemstrs(itemstrs, itemsep, newlines, _leaf_info, nobraces,<EOL>trailing_sep, compact_brace, lbr, rbr):

<EOL>use_newline = newlines > <NUM_LIT:0><EOL>if newlines < <NUM_LIT:0>:<EOL><INDENT>use_newline = (-newlines) < _leaf_info['<STR_LIT>']<EOL><DEDENT>if use_newline:<EOL><INDENT>sep = '<STR_LIT>'<EOL>if nobraces:<EOL><INDENT>body_str = sep.join(itemstrs)<EOL>if trailing_sep and len(itemstrs) > <NUM_LIT:0>:<EOL><INDENT>body_str += '<STR_LIT:U+002C>'<EOL><DEDENT>retstr = body_str<EOL><DEDENT>else:<EOL><INDENT>if compact_brace:<EOL><INDENT>indented = itemstrs<EOL><DEDENT>else:<EOL><INDENT>import ubelt as ub<EOL>prefix = '<STR_LIT:U+0020>' * <NUM_LIT:4><EOL>indented = [ub.indent(s, prefix) for s in itemstrs]<EOL><DEDENT>body_str = sep.join(indented)<EOL>if trailing_sep and len(itemstrs) > <NUM_LIT:0>:<EOL><INDENT>body_str += '<STR_LIT:U+002C>'<EOL><DEDENT>if compact_brace:<EOL><INDENT>braced_body_str = (lbr + body_str.replace('<STR_LIT:\n>', '<STR_LIT>') + rbr)<EOL><DEDENT>else:<EOL><INDENT>braced_body_str = (lbr + '<STR_LIT:\n>' + body_str + '<STR_LIT:\n>' + rbr)<EOL><DEDENT>retstr = braced_body_str<EOL><DEDENT><DEDENT>else:<EOL><INDENT>sep = '<STR_LIT:U+002C>' + itemsep<EOL>body_str = sep.join(itemstrs)<EOL>if trailing_sep and len(itemstrs) > <NUM_LIT:0>:<EOL><INDENT>body_str += '<STR_LIT:U+002C>'<EOL><DEDENT>retstr  = (lbr + body_str +  rbr)<EOL><DEDENT>return retstr<EOL>

Joins string-ified items with separators newlines and container-braces.


custom_extensions = kwargs.get('<STR_LIT>', None)<EOL>_return_info = kwargs.get('<STR_LIT>', False)<EOL>kwargs['<STR_LIT>'] = _rectify_root_info(kwargs.get('<STR_LIT>', None))<EOL>outstr = None<EOL>_leaf_info = None<EOL>if custom_extensions:<EOL><INDENT>func = custom_extensions.lookup(data)<EOL>if func is not None:<EOL><INDENT>outstr = func(data, **kwargs)<EOL><DEDENT><DEDENT>if outstr is None:<EOL><INDENT>if isinstance(data, dict):<EOL><INDENT>outstr, _leaf_info = _format_dict(data, **kwargs)<EOL><DEDENT>elif isinstance(data, (list, tuple, set, frozenset)):<EOL><INDENT>outstr, _leaf_info = _format_list(data, **kwargs)<EOL><DEDENT><DEDENT>if outstr is None:<EOL><INDENT>func = _FORMATTER_EXTENSIONS.lookup(data)<EOL>if func is not None:<EOL><INDENT>outstr = func(data, **kwargs)<EOL><DEDENT>else:<EOL><INDENT>outstr = _format_object(data, **kwargs)<EOL><DEDENT><DEDENT>if _return_info:<EOL><INDENT>_leaf_info = _rectify_leaf_info(_leaf_info)<EOL>return outstr, _leaf_info<EOL><DEDENT>else:<EOL><INDENT>return outstr<EOL><DEDENT>

Makes a pretty and easy-to-doctest string representation!

This is an alternative to repr, and `pprint.pformat` that attempts to be
both more configurable and generate output that is consistent between
python versions.

Notes:
    This function has many keyword arguments that can be used to customize
    the final representation. For convinience some of the more frequently
    used kwargs have short aliases. See `Args` for more details.

Args:
    data (object): an arbitrary python object
    **kwargs: see `the Kwargs` section

Kwargs:
    si, stritems, (bool):
        dict/list items use str instead of repr

    strkeys, sk (bool):
        dict keys use str instead of repr

    strvals, sv (bool):
        dict values use str instead of repr

    nl, newlines (int | bool):
        number of top level nestings to place a newline after. If true all
        items are followed by newlines regardless of nesting level.
        Defaults to 1 for lists and True for dicts.

    nobr, nobraces (bool, default=False):
        if True, text will not contain outer braces for containers

    cbr, compact_brace (bool, default=False):
        if True, braces are compactified (i.e. they will not have newlines
        placed directly after them, think java / K&R / 1TBS)

    trailsep, trailing_sep (bool):
        if True, a separator is placed after the last item in a sequence.
        By default this is True if there are any `nl > 0`.

    explicit (bool, default=False):
        changes dict representation from `{k1: v1, ...}` to
        `dict(k1=v1, ...)`.

    precision (int, default=None):
        if specified floats are formatted with this precision

    kvsep (str, default=': '):
        separator between keys and values

    itemsep (str, default=' '):
        separator between items

    sort (bool):
        if True, attempts to sort all unordered collections in the returned
        text. NOTE: currently if True this will sort lists, this may not be
        a correct thing to do, as such the behavior of this arg is subject
        to change.

    suppress_small (bool):
        passed to `numpy.array2string` for ndarrays

    max_line_width (int):
        passed to `numpy.array2string` for ndarrays

    with_dtype (bool):
        only relevant to ndarrays. if True includes the dtype.

Returns:
    str: outstr: output string

Notes:
    There are also internal kwargs, which should not be used:
        _return_info (bool):  return information about child context
        _root_info (depth): information about parent context

CommandLine:
    python -m ubelt.util_format repr2:0
    python -m ubelt.util_format repr2:1

Example:
    >>> from ubelt.util_format import *
    >>> import ubelt as ub
    >>> dict_ = {
    ...     'custom_types': [slice(0, 1, None), 1/3],
    ...     'nest_dict': {'k1': [1, 2, {3: {4, 5}}],
    ...                   'key2': [1, 2, {3: {4, 5}}],
    ...                   'key3': [1, 2, {3: {4, 5}}],
    ...                   },
    ...     'nest_dict2': {'k': [1, 2, {3: {4, 5}}]},
    ...     'nested_tuples': [tuple([1]), tuple([2, 3]), frozenset([4, 5, 6])],
    ...     'one_tup': tuple([1]),
    ...     'simple_dict': {'spam': 'eggs', 'ham': 'jam'},
    ...     'simple_list': [1, 2, 'red', 'blue'],
    ...     'odict': ub.odict([(1, '1'), (2, '2')]),
    ... }
    >>> result = repr2(dict_, nl=3, precision=2); print(result)
    >>> result = repr2(dict_, nl=2, precision=2); print(result)
    >>> result = repr2(dict_, nl=1, precision=2); print(result)
    >>> result = repr2(dict_, nl=1, precision=2, itemsep='', explicit=True); print(result)
    >>> result = repr2(dict_, nl=1, precision=2, nobr=1, itemsep='', explicit=True); print(result)
    >>> result = repr2(dict_, nl=3, precision=2, cbr=True); print(result)
    >>> result = repr2(dict_, nl=3, precision=2, si=True); print(result)
    >>> result = repr2(dict_, nl=3, sort=True); print(result)
    >>> result = repr2(dict_, nl=3, sort=False, trailing_sep=False); print(result)
    >>> result = repr2(dict_, nl=3, sort=False, trailing_sep=False, nobr=True); print(result)

Example:
    >>> from ubelt.util_format import *
    >>> def _nest(d, w):
    ...     if d == 0:
    ...         return {}
    ...     else:
    ...         return {'n{}'.format(d): _nest(d - 1, w + 1), 'm{}'.format(d): _nest(d - 1, w + 1)}
    >>> dict_ = _nest(d=4, w=1)
    >>> result = repr2(dict_, nl=6, precision=2, cbr=1)
    >>> print('---')
    >>> print(result)
    >>> result = repr2(dict_, nl=-1, precision=2)
    >>> print('---')
    >>> print(result)


def _decorator(func):<EOL><INDENT>if isinstance(type, tuple):<EOL><INDENT>for t in type:<EOL><INDENT>self.func_registry[t] = func<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.func_registry[type] = func<EOL><DEDENT>return func<EOL><DEDENT>return _decorator<EOL>

Registers a custom formatting function with ub.repr2


import numpy as np<EOL>@self.register(np.ndarray)<EOL>def format_ndarray(data, **kwargs):<EOL><INDENT>import re<EOL>strvals = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', False))<EOL>itemsep = kwargs.get('<STR_LIT>', '<STR_LIT:U+0020>')<EOL>precision = kwargs.get('<STR_LIT>', None)<EOL>suppress_small = kwargs.get('<STR_LIT>', None)<EOL>max_line_width = kwargs.get('<STR_LIT>', None)<EOL>with_dtype = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', not strvals))<EOL>newlines = kwargs.pop('<STR_LIT>', kwargs.pop('<STR_LIT>', <NUM_LIT:1>))<EOL>separator = '<STR_LIT:U+002C>' + itemsep<EOL>if strvals:<EOL><INDENT>prefix = '<STR_LIT>'<EOL>suffix = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>modname = type(data).__module__<EOL>np_nice = '<STR_LIT>'<EOL>modname = re.sub('<STR_LIT>', np_nice, modname)<EOL>modname = re.sub('<STR_LIT>', '<STR_LIT>', modname)<EOL>class_name = type(data).__name__<EOL>if class_name == '<STR_LIT>':<EOL><INDENT>class_name = '<STR_LIT>'<EOL><DEDENT>prefix = modname + '<STR_LIT:.>' + class_name + '<STR_LIT:(>'<EOL>if with_dtype:<EOL><INDENT>dtype_repr = data.dtype.name<EOL>suffix = '<STR_LIT>'.format(itemsep, np_nice, dtype_repr)<EOL><DEDENT>else:<EOL><INDENT>suffix = '<STR_LIT:)>'<EOL><DEDENT><DEDENT>if not strvals and data.size == <NUM_LIT:0> and data.shape != (<NUM_LIT:0>,):<EOL><INDENT>prefix = modname + '<STR_LIT>'<EOL>body = repr(tuple(map(int, data.shape)))<EOL><DEDENT>else:<EOL><INDENT>body = np.array2string(data, precision=precision,<EOL>separator=separator,<EOL>suppress_small=suppress_small,<EOL>prefix=prefix,<EOL>max_line_width=max_line_width)<EOL><DEDENT>if not newlines:<EOL><INDENT>body = re.sub('<STR_LIT>', '<STR_LIT>', body)<EOL><DEDENT>formatted = prefix + body + suffix<EOL>return formatted<EOL><DEDENT>self.register(np.float32)(self.func_registry[float])<EOL>

CommandLine:
    python -m ubelt.util_format FormatterExtensions._register_numpy_extensions

Example:
    >>> import sys
    >>> import pytest
    >>> import ubelt as ub
    >>> if not ub.modname_to_modpath('numpy'):
    ...     raise pytest.skip()
    >>> # xdoctest: +IGNORE_WHITESPACE
    >>> import numpy as np
    >>> data = np.array([[.2, 42, 5], [21.2, 3, .4]])
    >>> print(ub.repr2(data))
    np.array([[ 0.2, 42. ,  5. ],
              [21.2,  3. ,  0.4]], dtype=np.float64)
    >>> print(ub.repr2(data, with_dtype=False))
    np.array([[ 0.2, 42. ,  5. ],
              [21.2,  3. ,  0.4]])
    >>> print(ub.repr2(data, strvals=True))
    [[ 0.2, 42. ,  5. ],
     [21.2,  3. ,  0.4]]
    >>> data = np.empty((0, 10), dtype=np.float64)
    >>> print(ub.repr2(data, strvals=False))
    np.empty((0, 10), dtype=np.float64)
    >>> print(ub.repr2(data, strvals=True))
    []
    >>> data = np.ma.empty((0, 10), dtype=np.float64)
    >>> print(ub.repr2(data, strvals=False))
    np.ma.empty((0, 10), dtype=np.float64)


items = list(list_)<EOL>kwargs['<STR_LIT>'] = True<EOL>_tups = [repr2(item, **kwargs) for item in items]<EOL>itemstrs = [t[<NUM_LIT:0>] for t in _tups]<EOL>max_height = max([t[<NUM_LIT:1>]['<STR_LIT>'] for t in _tups]) if _tups else <NUM_LIT:0><EOL>_leaf_info = {<EOL>'<STR_LIT>': max_height + <NUM_LIT:1>,<EOL>}<EOL>sort = kwargs.get('<STR_LIT>', None)<EOL>if sort is None:<EOL><INDENT>sort = isinstance(list_, (set, frozenset))<EOL><DEDENT>if sort:<EOL><INDENT>itemstrs = _sort_itemstrs(items, itemstrs)<EOL><DEDENT>return itemstrs, _leaf_info<EOL>

Create a string representation for each item in a list.


import ubelt as ub<EOL>explicit = kwargs.get('<STR_LIT>', False)<EOL>kwargs['<STR_LIT>'] = _rectify_countdown_or_bool(explicit)<EOL>precision = kwargs.get('<STR_LIT>', None)<EOL>kvsep = kwargs.get('<STR_LIT>', '<STR_LIT>')<EOL>if explicit:<EOL><INDENT>kvsep = '<STR_LIT:=>'<EOL><DEDENT>def make_item_str(key, val):<EOL><INDENT>if explicit or kwargs.get('<STR_LIT>', False):<EOL><INDENT>key_str = six.text_type(key)<EOL><DEDENT>else:<EOL><INDENT>key_str = repr2(key, precision=precision, newlines=<NUM_LIT:0>)<EOL><DEDENT>prefix = key_str + kvsep<EOL>kwargs['<STR_LIT>'] = True<EOL>val_str, _leaf_info = repr2(val, **kwargs)<EOL>pos = val_str.find('<STR_LIT:\n>')<EOL>first_line = val_str if pos == -<NUM_LIT:1> else val_str[:pos]<EOL>compact_brace = kwargs.get('<STR_LIT>', kwargs.get('<STR_LIT>', False))<EOL>if compact_brace or not first_line.rstrip().endswith(tuple('<STR_LIT>')):<EOL><INDENT>rest = '<STR_LIT>' if pos == -<NUM_LIT:1> else val_str[pos:]<EOL>val_str = first_line.lstrip() + rest<EOL>if '<STR_LIT:\n>' in prefix:<EOL><INDENT>item_str = prefix + val_str<EOL><DEDENT>else:<EOL><INDENT>item_str = ub.hzcat([prefix, val_str])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>item_str = prefix + val_str<EOL><DEDENT>return item_str, _leaf_info<EOL><DEDENT>items = list(six.iteritems(dict_))<EOL>_tups = [make_item_str(key, val) for (key, val) in items]<EOL>itemstrs = [t[<NUM_LIT:0>] for t in _tups]<EOL>max_height = max([t[<NUM_LIT:1>]['<STR_LIT>'] for t in _tups]) if _tups else <NUM_LIT:0><EOL>_leaf_info = {<EOL>'<STR_LIT>': max_height + <NUM_LIT:1>,<EOL>}<EOL>sort = kwargs.get('<STR_LIT>', None)<EOL>if sort is None:<EOL><INDENT>sort = True<EOL><DEDENT>if isinstance(dict_, collections.OrderedDict):<EOL><INDENT>sort = False<EOL><DEDENT>if sort:<EOL><INDENT>itemstrs = _sort_itemstrs(items, itemstrs)<EOL><DEDENT>return itemstrs, _leaf_info<EOL>

Create a string representation for each item in a dict.

Example:
    >>> from ubelt.util_format import *
    >>> dict_ =  {'b': .1, 'l': 'st', 'g': 1.0, 's': 10, 'm': 0.9, 'w': .5}
    >>> kwargs = {'strkeys': True}
    >>> itemstrs, _ = _dict_itemstrs(dict_, **kwargs)
    >>> char_order = [p[0] for p in itemstrs]
    >>> assert char_order == ['b', 'g', 'l', 'm', 's', 'w']


kwitems = kwargs.items()<EOL>if (sys.version_info.major, sys.version_info.minor) < (<NUM_LIT:3>, <NUM_LIT:7>):  <EOL><INDENT>kwitems = sorted(kwitems)<EOL><DEDENT>kwitems = tuple(kwitems)<EOL>try:<EOL><INDENT>key = _hashable(args), _hashable(kwitems)<EOL><DEDENT>except TypeError:<EOL><INDENT>raise TypeError('<STR_LIT>'.format(args, kwargs))<EOL><DEDENT>return key<EOL>

Transforms function args into a key that can be used by the cache

CommandLine:
    xdoctest -m ubelt.util_memoize _make_signature_key

Example:
    >>> args = (4, [1, 2])
    >>> kwargs = {'a': 'b'}
    >>> key = _make_signature_key(args, kwargs)
    >>> print('key = {!r}'.format(key))
    >>> # Some mutable types cannot be handled by ub.hash_data
    >>> import pytest
    >>> import six
    >>> if six.PY2:
    >>>     import collections as abc
    >>> else:
    >>>     from collections import abc
    >>> with pytest.raises(TypeError):
    >>>     _make_signature_key((4, [1, 2], {1: 2, 'a': 'b'}), kwargs={})
    >>> class Dummy(abc.MutableSet):
    >>>     def __contains__(self, item): return None
    >>>     def __iter__(self): return iter([])
    >>>     def __len__(self): return 0
    >>>     def add(self, item, loc): return None
    >>>     def discard(self, item): return None
    >>> with pytest.raises(TypeError):
    >>>     _make_signature_key((Dummy(),), kwargs={})


cache = {}<EOL>@functools.wraps(func)<EOL>def memoizer(*args, **kwargs):<EOL><INDENT>key = _make_signature_key(args, kwargs)<EOL>if key not in cache:<EOL><INDENT>cache[key] = func(*args, **kwargs)<EOL><DEDENT>return cache[key]<EOL><DEDENT>memoizer.cache = cache<EOL>return memoizer<EOL>

memoization decorator that respects args and kwargs

References:
    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize

Args:
    func (Callable): live python function

Returns:
    func: memoized wrapper

CommandLine:
    xdoctest -m ubelt.util_memoize memoize

Example:
    >>> import ubelt as ub
    >>> closure = {'a': 'b', 'c': 'd'}
    >>> incr = [0]
    >>> def foo(key):
    >>>     value = closure[key]
    >>>     incr[0] += 1
    >>>     return value
    >>> foo_memo = ub.memoize(foo)
    >>> assert foo('a') == 'b' and foo('c') == 'd'
    >>> assert incr[0] == 2
    >>> print('Call memoized version')
    >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
    >>> assert incr[0] == 4
    >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'
    >>> print('Counter should no longer increase')
    >>> assert incr[0] == 4
    >>> print('Closure changes result without memoization')
    >>> closure = {'a': 0, 'c': 1}
    >>> assert foo('a') == 0 and foo('c') == 1
    >>> assert incr[0] == 6
    >>> assert foo_memo('a') == 'b' and foo_memo('c') == 'd'


def _syspath_modname_to_modpath(modname, sys_path=None, exclude=None):

def _isvalid(modpath, base):<EOL><INDENT>subdir = dirname(modpath)<EOL>while subdir and subdir != base:<EOL><INDENT>if not exists(join(subdir, '<STR_LIT>')):<EOL><INDENT>return False<EOL><DEDENT>subdir = dirname(subdir)<EOL><DEDENT>return True<EOL><DEDENT>_fname_we = modname.replace('<STR_LIT:.>', os.path.sep)<EOL>candidate_fnames = [<EOL>_fname_we + '<STR_LIT>',<EOL>]<EOL>candidate_fnames += [_fname_we + ext for ext in _platform_pylib_exts()]<EOL>if sys_path is None:<EOL><INDENT>sys_path = sys.path<EOL><DEDENT>candidate_dpaths = ['<STR_LIT:.>' if p == '<STR_LIT>' else p for p in sys_path]<EOL>if exclude:<EOL><INDENT>def normalize(p):<EOL><INDENT>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>return realpath(p).lower()<EOL><DEDENT>else:<EOL><INDENT>return realpath(p)<EOL><DEDENT><DEDENT>real_exclude = {normalize(p) for p in exclude}<EOL>candidate_dpaths = [p for p in candidate_dpaths<EOL>if normalize(p) not in real_exclude]<EOL><DEDENT>for dpath in candidate_dpaths:<EOL><INDENT>modpath = join(dpath, _fname_we)<EOL>if exists(modpath):<EOL><INDENT>if isfile(join(modpath, '<STR_LIT>')):<EOL><INDENT>if _isvalid(modpath, dpath):<EOL><INDENT>return modpath<EOL><DEDENT><DEDENT><DEDENT>for fname in candidate_fnames:<EOL><INDENT>modpath = join(dpath, fname)<EOL>if isfile(modpath):<EOL><INDENT>if _isvalid(modpath, dpath):<EOL><INDENT>return modpath<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

syspath version of modname_to_modpath

Args:
    modname (str): name of module to find
    sys_path (List[PathLike], default=None):
        if specified overrides `sys.path`
    exclude (List[PathLike], default=None):
        list of directory paths. if specified prevents these directories
        from being searched.

Notes:
    This is much slower than the pkgutil mechanisms.

CommandLine:
    python -m xdoctest.static_analysis _syspath_modname_to_modpath

Example:
    >>> print(_syspath_modname_to_modpath('xdoctest.static_analysis'))
    ...static_analysis.py
    >>> print(_syspath_modname_to_modpath('xdoctest'))
    ...xdoctest
    >>> print(_syspath_modname_to_modpath('_ctypes'))
    ..._ctypes...
    >>> assert _syspath_modname_to_modpath('xdoctest', sys_path=[]) is None
    >>> assert _syspath_modname_to_modpath('xdoctest.static_analysis', sys_path=[]) is None
    >>> assert _syspath_modname_to_modpath('_ctypes', sys_path=[]) is None
    >>> assert _syspath_modname_to_modpath('this', sys_path=[]) is None

Example:
    >>> # test what happens when the module is not visible in the path
    >>> modname = 'xdoctest.static_analysis'
    >>> modpath = _syspath_modname_to_modpath(modname)
    >>> exclude = [split_modpath(modpath)[0]]
    >>> found = _syspath_modname_to_modpath(modname, exclude=exclude)
    >>> # this only works if installed in dev mode, pypi fails
    >>> assert found is None, 'should not have found {}'.format(found)


if True:<EOL><INDENT>import importlib<EOL>module = importlib.import_module(modname)<EOL><DEDENT>else:<EOL><INDENT>if '<STR_LIT:.>' in modname:<EOL><INDENT>fromlist = modname.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>fromlist_ = list(map(str, fromlist))  <EOL>module = __import__(modname, {}, {}, fromlist_, <NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>module = __import__(modname, {}, {}, [], <NUM_LIT:0>)<EOL><DEDENT><DEDENT>return module<EOL>

Imports a module from its string name (__name__)

Args:
    modname (str):  module name

Returns:
    module: module

Example:
    >>> # test with modules that wont be imported in normal circumstances
    >>> # todo write a test where we gaurentee this
    >>> modname_list = [
    >>>     'pickletools',
    >>>     'lib2to3.fixes.fix_apply',
    >>> ]
    >>> #assert not any(m in sys.modules for m in modname_list)
    >>> modules = [import_module_from_name(modname) for modname in modname_list]
    >>> assert [m.__name__ for m in modules] == modname_list
    >>> assert all(m in sys.modules for m in modname_list)


if six.PY2:<EOL><INDENT>if modpath.endswith('<STR_LIT>'):<EOL><INDENT>modpath = modpath[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>modpath_ = abspath(expanduser(modpath))<EOL>if check:<EOL><INDENT>if not exists(modpath_):<EOL><INDENT>if not exists(modpath):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT>if isdir(modpath_) and not exists(join(modpath, '<STR_LIT>')):<EOL><INDENT>raise ValueError('<STR_LIT>'.format(modpath))<EOL><DEDENT><DEDENT>full_dpath, fname_ext = split(modpath_)<EOL>_relmod_parts = [fname_ext]<EOL>dpath = full_dpath<EOL>while exists(join(dpath, '<STR_LIT>')):<EOL><INDENT>dpath, dname = split(dpath)<EOL>_relmod_parts.append(dname)<EOL><DEDENT>relmod_parts = _relmod_parts[::-<NUM_LIT:1>]<EOL>rel_modpath = os.path.sep.join(relmod_parts)<EOL>return dpath, rel_modpath<EOL>

Splits the modpath into the dir that must be in PYTHONPATH for the module
to be imported and the modulepath relative to this directory.

Args:
    modpath (str): module filepath
    check (bool): if False, does not raise an error if modpath is a
        directory and does not contain an `__init__.py` file.

Returns:
    tuple: (directory, rel_modpath)

Raises:
    ValueError: if modpath does not exist or is not a package

Example:
    >>> from xdoctest import static_analysis
    >>> modpath = static_analysis.__file__.replace('.pyc', '.py')
    >>> modpath = abspath(modpath)
    >>> dpath, rel_modpath = split_modpath(modpath)
    >>> recon = join(dpath, rel_modpath)
    >>> assert recon == modpath
    >>> assert rel_modpath == join('xdoctest', 'static_analysis.py')


def modname_to_modpath(modname, hide_init=True, hide_main=False, sys_path=None):

modpath = _syspath_modname_to_modpath(modname, sys_path)<EOL>if modpath is None:<EOL><INDENT>return None<EOL><DEDENT>modpath = normalize_modpath(modpath, hide_init=hide_init,<EOL>hide_main=hide_main)<EOL>return modpath<EOL>

Finds the path to a python module from its name.

Determines the path to a python module without directly import it

Converts the name of a module (__name__) to the path (__file__) where it is
located without importing the module. Returns None if the module does not
exist.

Args:
    modname (str): module filepath
    hide_init (bool): if False, __init__.py will be returned for packages
    hide_main (bool): if False, and hide_init is True, __main__.py will be
        returned for packages, if it exists.
    sys_path (list): if specified overrides `sys.path` (default None)

Returns:
    str: modpath - path to the module, or None if it doesn't exist

CommandLine:
    python -m xdoctest.static_analysis modname_to_modpath:0
    pytest  /home/joncrall/code/xdoctest/xdoctest/static_analysis.py::modname_to_modpath:0

Example:
    >>> modname = 'xdoctest.__main__'
    >>> modpath = modname_to_modpath(modname, hide_main=False)
    >>> assert modpath.endswith('__main__.py')
    >>> modname = 'xdoctest'
    >>> modpath = modname_to_modpath(modname, hide_init=False)
    >>> assert modpath.endswith('__init__.py')
    >>> modpath = basename(modname_to_modpath('_ctypes'))
    >>> assert 'ctypes' in modpath


import os<EOL>if not os.path.exists(modpath):<EOL><INDENT>import re<EOL>import zipimport<EOL>pat = '<STR_LIT>' + re.escape(os.path.sep) + '<STR_LIT>'<EOL>parts = re.split(pat, modpath, flags=re.IGNORECASE)<EOL>if len(parts) > <NUM_LIT:2>:<EOL><INDENT>archivepath = '<STR_LIT>'.join(parts[:-<NUM_LIT:1>])[:-<NUM_LIT:1>]<EOL>internal = parts[-<NUM_LIT:1>]<EOL>modname = os.path.splitext(internal)[<NUM_LIT:0>]<EOL>modname = os.path.normpath(modname)<EOL>if os.path.exists(archivepath):<EOL><INDENT>zimp_file = zipimport.zipimporter(archivepath)<EOL>module = zimp_file.load_module(modname)<EOL>return module<EOL><DEDENT><DEDENT>raise IOError('<STR_LIT>'.format(modpath))<EOL><DEDENT>else:<EOL><INDENT>module = _custom_import_modpath(modpath)<EOL>return module<EOL><DEDENT>

Imports a module via its path

Args:
    modpath (PathLike): path to the module on disk or within a zipfile.

Returns:
    module: the imported module

References:
    https://stackoverflow.com/questions/67631/import-module-given-path

Notes:
    If the module is part of a package, the package will be imported first.
    These modules may cause problems when reloading via IPython magic

    This can import a module from within a zipfile. To do this modpath
    should specify the path to the zipfile and the path to the module
    within that zipfile separated by a colon or pathsep.
    E.g. `/path/to/archive.zip:mymodule.py`

Warning:
    It is best to use this with paths that will not conflict with
    previously existing modules.

    If the modpath conflicts with a previously existing module name. And
    the target module does imports of its own relative to this conflicting
    path. In this case, the module that was loaded first will win.

    For example if you try to import '/foo/bar/pkg/mod.py' from the folder
    structure:
      - foo/
        +- bar/
           +- pkg/
              +  __init__.py
              |- mod.py
              |- helper.py

   If there exists another module named `pkg` already in sys.modules
   and mod.py does something like `from . import helper`, Python will
   assume helper belongs to the `pkg` module already in sys.modules.
   This can cause a NameError or worse --- a incorrect helper module.

Example:
    >>> import xdoctest
    >>> modpath = xdoctest.__file__
    >>> module = import_module_from_path(modpath)
    >>> assert module is xdoctest

Example:
    >>> # Test importing a module from within a zipfile
    >>> import zipfile
    >>> from xdoctest import utils
    >>> from os.path import join, expanduser
    >>> dpath = expanduser('~/.cache/xdoctest')
    >>> dpath = utils.ensuredir(dpath)
    >>> #dpath = utils.TempDir().ensure()
    >>> # Write to an external module named bar
    >>> external_modpath = join(dpath, 'bar.py')
    >>> open(external_modpath, 'w').write('testvar = 1')
    >>> internal = 'folder/bar.py'
    >>> # Move the external bar module into a zipfile
    >>> zippath = join(dpath, 'myzip.zip')
    >>> with zipfile.ZipFile(zippath, 'w') as myzip:
    >>>     myzip.write(external_modpath, internal)
    >>> # Import the bar module from within the zipfile
    >>> modpath = zippath + ':' + internal
    >>> modpath = zippath + os.path.sep + internal
    >>> module = import_module_from_path(modpath)
    >>> assert module.__name__ == os.path.normpath('folder/bar')
    >>> assert module.testvar == 1

Doctest:
    >>> import pytest
    >>> with pytest.raises(IOError):
    >>>     import_module_from_path('does-not-exist')
    >>> with pytest.raises(IOError):
    >>>     import_module_from_path('does-not-exist.zip/')


from ubelt import util_colors<EOL>if dpath is None:<EOL><INDENT>dpath = os.getcwd()<EOL><DEDENT>print('<STR_LIT>')<EOL>print('<STR_LIT>'.format(dpath))<EOL>print('<STR_LIT>')<EOL>print('<STR_LIT>')<EOL>if not os.path.exists(dpath):<EOL><INDENT>print('<STR_LIT>')<EOL>return<EOL><DEDENT>paths = sorted(os.listdir(dpath))<EOL>for path in paths:<EOL><INDENT>full_path = join(dpath, path)<EOL>E = os.path.exists(full_path)<EOL>L = os.path.islink(full_path)<EOL>F = os.path.isfile(full_path)<EOL>D = os.path.isdir(full_path)<EOL>J = util_platform.WIN32 and _win32_links._win32_is_junction(full_path)<EOL>ELFDJ = [E, L, F, D, J]<EOL>if   ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:1>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:1>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:1>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>elif ELFDJ == [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>]:<EOL><INDENT>path = util_colors.color_text(path, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>'.format(dpath))<EOL>print('<STR_LIT>'.format(path))<EOL>raise AssertionError(str(ELFDJ) + str(path))<EOL><DEDENT>line = '<STR_LIT>'.format(**locals())<EOL>if os.path.islink(full_path):<EOL><INDENT>line += '<STR_LIT>' + os.readlink(full_path)<EOL><DEDENT>elif _win32_links is not None:<EOL><INDENT>if _win32_links._win32_is_junction(full_path):<EOL><INDENT>line += '<STR_LIT>' + _win32_links._win32_read_junction(full_path)<EOL><DEDENT><DEDENT>print(line)<EOL><DEDENT>

Testing helper for printing directory information
(mostly for investigating windows weirdness)

CommandLine:
    python -m ubelt.util_links _dirstats


if _win32_links is not None:<EOL><INDENT>return _win32_links._win32_can_symlink(verbose)<EOL><DEDENT>else:<EOL><INDENT>return True<EOL><DEDENT>

Return true if we have permission to create real symlinks.
This check always returns True on non-win32 systems.
If this check returns false, then we still may be able to use junctions.

CommandLine:
    python -m ubelt.util_platform _can_symlink

Example:
    >>> # Script
    >>> print(_can_symlink(verbose=1))


if self.redirect is not None:<EOL><INDENT>return self.redirect.encoding<EOL><DEDENT>else:<EOL><INDENT>return super(TeeStringIO, self).encoding<EOL><DEDENT>

Gets the encoding of the `redirect` IO object

Doctest:
    >>> redirect = io.StringIO()
    >>> assert TeeStringIO(redirect).encoding is None
    >>> assert TeeStringIO(None).encoding is None
    >>> assert TeeStringIO(sys.stdout).encoding is sys.stdout.encoding
    >>> redirect = io.TextIOWrapper(io.StringIO())
    >>> assert TeeStringIO(redirect).encoding is redirect.encoding


if self.redirect is not None:<EOL><INDENT>self.redirect.write(msg)<EOL><DEDENT>if six.PY2:<EOL><INDENT>from xdoctest.utils.util_str import ensure_unicode<EOL>msg = ensure_unicode(msg)<EOL><DEDENT>super(TeeStringIO, self).write(msg)<EOL>


return (self.redirect is not None and<EOL>hasattr(self.redirect, '<STR_LIT>') and self.redirect.isatty())<EOL>

Returns true of the redirect is a terminal.

Notes:
    Needed for IPython.embed to work properly when this class is used
    to override stdout / stderr.


def ensuredir(dpath, mode=<NUM_LIT>, verbose=None):

if verbose is None:  <EOL><INDENT>verbose = <NUM_LIT:0><EOL><DEDENT>if isinstance(dpath, (list, tuple)):  <EOL><INDENT>dpath = join(*dpath)<EOL><DEDENT>if not exists(dpath):<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>' % dpath)<EOL><DEDENT>if sys.version_info.major == <NUM_LIT:2>:  <EOL><INDENT>os.makedirs(normpath(dpath), mode=mode)<EOL><DEDENT>else:<EOL><INDENT>os.makedirs(normpath(dpath), mode=mode, exist_ok=True)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>' % dpath)<EOL><DEDENT><DEDENT>return dpath<EOL>

r"""
    Ensures that directory will exist. Creates new dir with sticky bits by
    default

    Args:
        dpath (PathLike): dir to ensure. Can also be a tuple to send to join
        mode (int): octal mode of directory (default 0o1777)
        verbose (int): verbosity (default 0)

    Returns:
        PathLike: path: the ensured directory

    Notes:
        This function is not thread-safe in Python2

    Example:
        >>> from ubelt.util_platform import *  # NOQA
        >>> import ubelt as ub
        >>> cache_dpath = ub.ensure_app_cache_dir('ubelt')
        >>> dpath = join(cache_dpath, 'ensuredir')
        >>> if exists(dpath):
        ...     os.rmdir(dpath)
        >>> assert not exists(dpath)
        >>> ub.ensuredir(dpath)
        >>> assert exists(dpath)
        >>> os.rmdir(dpath)


path = expanduser(path)<EOL>path = expandvars(path)<EOL>if real:<EOL><INDENT>path = realpath(path)<EOL><DEDENT>else:<EOL><INDENT>path = abspath(path)<EOL><DEDENT>path = normpath(path)<EOL>return path<EOL>

Normalizes a string representation of a path and does shell-like expansion.

Args:
    path (PathLike): string representation of a path
    real (bool): if True, all symbolic links are followed. (default: False)

Returns:
    PathLike : normalized path

Note:
    This function is similar to the composition of expanduser, expandvars,
    normpath, and (realpath if `real` else abspath). However, on windows
    backslashes are then replaced with forward slashes to offer a
    consistent unix-like experience across platforms.

    On windows expanduser will expand environment variables formatted as
    %name%, whereas on unix, this will not occur.

CommandLine:
    python -m ubelt.util_path truepath

Example:
    >>> import ubelt as ub
    >>> assert ub.truepath('~/foo') == join(ub.userhome(), 'foo')
    >>> assert ub.truepath('~/foo') == ub.truepath('~/foo/bar/..')
    >>> assert ub.truepath('~/foo', real=True) == ub.truepath('~/foo')


path = expanduser(path)<EOL>path = expandvars(path)<EOL>return path<EOL>

Wrapper around expanduser and expandvars.

Less aggressive than truepath. Only expands environs and tilde. Does not
change relative paths to absolute paths.

Args:
    path (PathLike): string representation of a path

Returns:
    PathLike : expanded path

Example:
    >>> import ubelt as ub
    >>> assert normpath(ub.expandpath('~/foo')) == join(ub.userhome(), 'foo')
    >>> assert ub.expandpath('foo') == 'foo'


def download(url, fpath=None, hash_prefix=None, hasher='<STR_LIT>',<EOL>chunksize=<NUM_LIT>, verbose=<NUM_LIT:1>):

from progiter import ProgIter as Progress<EOL>from ubelt import util_platform<EOL>import shutil<EOL>import tempfile<EOL>import hashlib<EOL>if six.PY2:  <EOL><INDENT>from urllib2 import urlopen  <EOL><DEDENT>else:<EOL><INDENT>from urllib.request import urlopen  <EOL><DEDENT>if fpath is None:<EOL><INDENT>dpath = util_platform.ensure_app_cache_dir('<STR_LIT>')<EOL>fname = basename(url)<EOL>fpath = join(dpath, fname)<EOL><DEDENT>_dst_is_io_object = hasattr(fpath, '<STR_LIT>')<EOL>if verbose:<EOL><INDENT>if _dst_is_io_object:<EOL><INDENT>print('<STR_LIT>' % (url,))<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>' % (url, fpath))<EOL><DEDENT><DEDENT>urldata = urlopen(url)<EOL>meta = urldata.info()<EOL>try:<EOL><INDENT>if hasattr(meta, '<STR_LIT>'):  <EOL><INDENT>file_size = int(meta.getheaders("<STR_LIT>")[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>file_size = int(meta.get_all("<STR_LIT>")[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>except Exception:  <EOL><INDENT>file_size = None<EOL><DEDENT>if hash_prefix:<EOL><INDENT>if isinstance(hasher, six.string_types):<EOL><INDENT>if hasher == '<STR_LIT>':<EOL><INDENT>hasher = hashlib.sha1()<EOL><DEDENT>elif hasher == '<STR_LIT>':<EOL><INDENT>hasher = hashlib.sha512()<EOL><DEDENT>else:<EOL><INDENT>raise KeyError(hasher)<EOL><DEDENT><DEDENT><DEDENT>if _dst_is_io_object:<EOL><INDENT>_file_write = fpath.write<EOL><DEDENT>else:<EOL><INDENT>tmp = tempfile.NamedTemporaryFile(delete=False)<EOL>_file_write = tmp.write<EOL><DEDENT>_urldata_read = urldata.read<EOL>try:<EOL><INDENT>with Progress(total=file_size, disable=not verbose) as pbar:<EOL><INDENT>_pbar_update = pbar.update<EOL>def _critical_loop():<EOL><INDENT>buffer = '<STR_LIT:U+0020>'<EOL>if hash_prefix:<EOL><INDENT>_hasher_update = hasher.update<EOL>while buffer:<EOL><INDENT>buffer = _urldata_read(chunksize)<EOL>_file_write(buffer)<EOL>_hasher_update(buffer)<EOL>_pbar_update(len(buffer))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>while buffer:<EOL><INDENT>buffer = _urldata_read(chunksize)<EOL>_file_write(buffer)<EOL>_pbar_update(len(buffer))<EOL><DEDENT><DEDENT><DEDENT>_critical_loop()<EOL><DEDENT>if not _dst_is_io_object:<EOL><INDENT>tmp.close()<EOL>shutil.move(tmp.name, fpath)<EOL><DEDENT>if hash_prefix:<EOL><INDENT>got = hasher.hexdigest()<EOL>if got[:len(hash_prefix)] != hash_prefix:<EOL><INDENT>print('<STR_LIT>'.format(hash_prefix))<EOL>print('<STR_LIT>'.format(got))<EOL>if _dst_is_io_object:<EOL><INDENT>raise RuntimeError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(hash_prefix, got))<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>fpath, hash_prefix, got))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>finally:<EOL><INDENT>if not _dst_is_io_object:  <EOL><INDENT>tmp.close()<EOL>if exists(tmp.name):<EOL><INDENT>os.remove(tmp.name)<EOL><DEDENT><DEDENT><DEDENT>return fpath<EOL>

downloads a url to a fpath.

Args:
    url (str):
        The url to download.

    fpath (PathLike | io.BytesIOtringIO):
        The path to download to. Defaults to basename of url and ubelt's
        application cache. If this is a io.BytesIO object then information
        is directly written to this object (note this prevents the use of
        temporary files).

    hash_prefix (None or str):
        If specified, download will retry / error if the file hash
        does not match this value. Defaults to None.

    hasher (str or Hasher):
        If hash_prefix is specified, this indicates the hashing
        algorithm to apply to the file. Defaults to sha512.

    chunksize (int):
        Download chunksize. Defaults to 2 ** 13.

    verbose (int):
        Verbosity level 0 or 1. Defaults to 1.

Returns:
    PathLike: fpath - file path string

Raises:
    URLError - if there is problem downloading the url
    RuntimeError - if the hash does not match the hash_prefix

Notes:
    Original code taken from pytorch in torch/utils/model_zoo.py and
    slightly modified.

References:
    http://blog.moleculea.com/2012/10/04/urlretrieve-progres-indicator/
    http://stackoverflow.com/questions/15644964/python-progress-bar-and-downloads
    http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py

CommandLine:
    python -m xdoctest ubelt.util_download download:1

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> from ubelt.util_download import *  # NOQA
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> fpath = download(url)
    >>> print(basename(fpath))
    rqwaDag.png

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> import ubelt as ub
    >>> import io
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> file = io.BytesIO()
    >>> fpath = download(url, file)
    >>> file.seek(0)
    >>> data = file.read()
    >>> assert ub.hash_data(data, hasher='sha1').startswith('f79ea24571')

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> url = 'http://i.imgur.com/rqwaDag.png'
    >>> fpath = download(url, hasher='sha1', hash_prefix='f79ea24571da6ddd2ba12e3d57b515249ecb8a35')
    Downloading url='http://i.imgur.com/rqwaDag.png' to fpath=...rqwaDag.png
    ...
    ...1233/1233... rate=... Hz, eta=..., total=..., wall=...

Example:
    >>> # xdoctest: +REQUIRES(--network)
    >>> # test download from girder
    >>> import pytest
    >>> import ubelt as ub
    >>> url = 'https://data.kitware.com/api/v1/item/5b4039308d777f2e6225994c/download'
    >>> ub.download(url, hasher='sha512', hash_prefix='c98a46cb31205cf')
    >>> with pytest.raises(RuntimeError):
    >>>     ub.download(url, hasher='sha512', hash_prefix='BAD_HASH')


def cmd(command, shell=False, detach=False, verbose=<NUM_LIT:0>, tee=None, cwd=None,<EOL>env=None, tee_backend='<STR_LIT>', verbout=None, **kwargs):

if kwargs:  <EOL><INDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>warnings.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', DeprecationWarning)<EOL>tee = kwargs.pop('<STR_LIT>')<EOL><DEDENT>if '<STR_LIT>' in kwargs:<EOL><INDENT>warnings.warn(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', DeprecationWarning)<EOL>detach = kwargs.pop('<STR_LIT>')<EOL><DEDENT>if kwargs:<EOL><INDENT>raise ValueError('<STR_LIT>'.format(list(kwargs.keys)))<EOL><DEDENT><DEDENT>if isinstance(command, six.string_types):<EOL><INDENT>command_text = command<EOL>command_tup = None<EOL><DEDENT>else:<EOL><INDENT>import pipes<EOL>command_tup = command<EOL>command_text = '<STR_LIT:U+0020>'.join(list(map(pipes.quote, command_tup)))<EOL><DEDENT>if shell or sys.platform.startswith('<STR_LIT:win32>'):<EOL><INDENT>args = command_text<EOL><DEDENT>else:<EOL><INDENT>if command_tup is None:<EOL><INDENT>import shlex<EOL>command_tup = shlex.split(command_text)<EOL><DEDENT>args = command_tup<EOL><DEDENT>if tee is None:<EOL><INDENT>tee = verbose > <NUM_LIT:0><EOL><DEDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>import os<EOL>import platform<EOL>import getpass<EOL>from ubelt import util_path<EOL>if verbose > <NUM_LIT:2>:<EOL><INDENT>try:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>except Exception:  <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT>cwd_ = os.getcwd() if cwd is None else cwd<EOL>compname = platform.node()<EOL>username = getpass.getuser()<EOL>cwd_ = util_path.compressuser(cwd_)<EOL>ps1 = '<STR_LIT>'.format(username, compname, cwd_)<EOL>print(ps1 + command_text)<EOL><DEDENT>def make_proc():<EOL><INDENT>import subprocess<EOL>proc = subprocess.Popen(args, stdout=subprocess.PIPE,<EOL>stderr=subprocess.PIPE, shell=shell,<EOL>universal_newlines=True, cwd=cwd, env=env)<EOL>return proc<EOL><DEDENT>if detach:<EOL><INDENT>info = {'<STR_LIT>': make_proc(), '<STR_LIT>': command_text}<EOL>if verbose > <NUM_LIT:0>:  <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if tee:<EOL><INDENT>stdout, stderr = sys.stdout, sys.stderr<EOL>proc, logged_out, logged_err = _tee_output(make_proc, stdout, stderr,<EOL>backend=tee_backend)<EOL>try:<EOL><INDENT>out = '<STR_LIT>'.join(logged_out)<EOL><DEDENT>except UnicodeDecodeError:  <EOL><INDENT>out = '<STR_LIT:\n>'.join(_.decode('<STR_LIT:utf-8>') for _ in logged_out)<EOL><DEDENT>try:<EOL><INDENT>err = '<STR_LIT>'.join(logged_err)<EOL><DEDENT>except UnicodeDecodeError:  <EOL><INDENT>err = '<STR_LIT:\n>'.join(_.decode('<STR_LIT:utf-8>') for _ in logged_err)<EOL><DEDENT>(out_, err_) = proc.communicate()<EOL><DEDENT>else:<EOL><INDENT>proc = make_proc()<EOL>(out, err) = proc.communicate()<EOL><DEDENT>ret = proc.wait()<EOL>info = {<EOL>'<STR_LIT>': out,<EOL>'<STR_LIT>': err,<EOL>'<STR_LIT>': ret,<EOL>'<STR_LIT>': proc,<EOL>'<STR_LIT>': cwd,<EOL>'<STR_LIT>': command_text<EOL>}<EOL>if verbose > <NUM_LIT:2>:<EOL><INDENT>try:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>except Exception:  <EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>return info<EOL>

Executes a command in a subprocess.

The advantage of this wrapper around subprocess is that
(1) you control if the subprocess prints to stdout,
(2) the text written to stdout and stderr is returned for parsing,
(3) cross platform behavior that lets you specify the command as a string
or tuple regardless of whether or not shell=True.
(4) ability to detach, return the process object and allow the process to
run in the background (eventually we may return a Future object instead).

Args:
    command (str or Sequence): bash-like command string or tuple of
        executable and args

    shell (bool): if True, process is run in shell, defaults to False.

    detach (bool): if True, process is detached and run in background,
        defaults to False.

    verbose (int): verbosity mode. Can be 0, 1, 2, or 3. Defaults to 0.

    tee (bool, optional): if True, simultaneously writes to stdout while
        capturing output from the command. If not specified, defaults to
        True if verbose > 0.  If detech is True, then this argument is
        ignored.

    cwd (PathLike, optional): path to run command

    env (str, optional): environment passed to Popen

    tee_backend (str, optional): backend for tee output.
        Valid choices are: "auto", "select" (POSIX only), and "thread".

    **kwargs: only used to support deprecated arguments

Returns:
    dict: info - information about command status.
        if detach is False `info` contains captured standard out,
        standard error, and the return code
        if detach is False `info` contains a reference to the process.

Notes:
    Inputs can either be text or tuple based. On UNIX we ensure conversion
    to text if shell=True, and to tuple if shell=False. On windows, the
    input is always text based.  See [3] for a potential cross-platform
    shlex solution for windows.

CommandLine:
    python -m ubelt.util_cmd cmd
    python -c "import ubelt as ub; ub.cmd('ping localhost -c 2', verbose=2)"

References:
    [1] https://stackoverflow.com/questions/11495783/redirect-subprocess-stderr-to-stdout
    [2] https://stackoverflow.com/questions/7729336/how-can-i-print-and-display-subprocess-stdout-and-stderr-output-without-distorti
    [3] https://stackoverflow.com/questions/33560364/python-windows-parsing-command-lines-with-shlex

Example:
    >>> info = cmd(('echo', 'simple cmdline interface'), verbose=1)
    simple cmdline interface
    >>> assert info['ret'] == 0
    >>> assert info['out'].strip() == 'simple cmdline interface'
    >>> assert info['err'].strip() == ''

Doctest:
    >>> info = cmd('echo str noshell', verbose=0)
    >>> assert info['out'].strip() == 'str noshell'

Doctest:
    >>> # windows echo will output extra single quotes
    >>> info = cmd(('echo', 'tuple noshell'), verbose=0)
    >>> assert info['out'].strip().strip("'") == 'tuple noshell'

Doctest:
    >>> # Note this command is formatted to work on win32 and unix
    >>> info = cmd('echo str&&echo shell', verbose=0, shell=True)
    >>> assert info['out'].strip() == 'str' + chr(10) + 'shell'

Doctest:
    >>> info = cmd(('echo', 'tuple shell'), verbose=0, shell=True)
    >>> assert info['out'].strip().strip("'") == 'tuple shell'

Doctest:
    >>> import ubelt as ub
    >>> from os.path import join, exists
    >>> fpath1 = join(ub.get_app_cache_dir('ubelt'), 'cmdout1.txt')
    >>> fpath2 = join(ub.get_app_cache_dir('ubelt'), 'cmdout2.txt')
    >>> ub.delete(fpath1)
    >>> ub.delete(fpath2)
    >>> info1 = ub.cmd(('touch', fpath1), detach=True)
    >>> info2 = ub.cmd('echo writing2 > ' + fpath2, shell=True, detach=True)
    >>> while not exists(fpath1):
    ...     pass
    >>> while not exists(fpath2):
    ...     pass
    >>> assert ub.readfrom(fpath1) == ''
    >>> assert ub.readfrom(fpath2).strip() == 'writing2'
    >>> info1['proc'].wait()
    >>> info2['proc'].wait()


from six.moves import queue<EOL>stdout_queue = _proc_async_iter_stream(proc, proc.stdout)<EOL>stderr_queue = _proc_async_iter_stream(proc, proc.stderr)<EOL>stdout_live = True<EOL>stderr_live = True<EOL>while stdout_live or stderr_live:<EOL><INDENT>if stdout_live:  <EOL><INDENT>try:<EOL><INDENT>oline = stdout_queue.get_nowait()<EOL>stdout_live = oline is not None<EOL><DEDENT>except queue.Empty:<EOL><INDENT>oline = None<EOL><DEDENT><DEDENT>if stderr_live:<EOL><INDENT>try:<EOL><INDENT>eline = stderr_queue.get_nowait()<EOL>stderr_live = eline is not None<EOL><DEDENT>except queue.Empty:<EOL><INDENT>eline = None<EOL><DEDENT><DEDENT>if oline is not None or eline is not None:<EOL><INDENT>yield oline, eline<EOL><DEDENT><DEDENT>

Iterates over output from a process line by line

Note:
    WARNING. Current implementation might have bugs with other threads.
    This behavior was seen when using earlier versions of tqdm. I'm not
    sure if this was our bug or tqdm's. Newer versions of tqdm fix this,
    but I cannot guarantee that there isn't an issue on our end.

Yields:
    Tuple[str, str]: oline, eline: stdout and stderr line

References:
    https://stackoverflow.com/questions/375427/non-blocking-read-subproc


from six.moves import zip_longest<EOL>while proc.poll() is None:<EOL><INDENT>reads = [proc.stdout.fileno(), proc.stderr.fileno()]<EOL>ret = select.select(reads, [], [])<EOL>oline = eline = None<EOL>for fd in ret[<NUM_LIT:0>]:<EOL><INDENT>if fd == proc.stdout.fileno():<EOL><INDENT>oline = proc.stdout.readline()<EOL><DEDENT>if fd == proc.stderr.fileno():<EOL><INDENT>eline = proc.stderr.readline()<EOL><DEDENT><DEDENT>yield oline, eline<EOL><DEDENT>oline_iter = _textio_iterlines(proc.stdout)<EOL>eline_iter = _textio_iterlines(proc.stderr)<EOL>for oline, eline in zip_longest(oline_iter, eline_iter):<EOL><INDENT>yield oline, eline<EOL><DEDENT>

Iterates over output from a process line by line

UNIX only. Use `_proc_iteroutput_thread` instead for a cross platform
solution based on threads.

Yields:
    Tuple[str, str]: oline, eline: stdout and stderr line


def _proc_async_iter_stream(proc, stream, buffersize=<NUM_LIT:1>):

from six.moves import queue<EOL>from threading import Thread<EOL>def enqueue_output(proc, stream, stream_queue):<EOL><INDENT>while proc.poll() is None:<EOL><INDENT>line = stream.readline()<EOL>stream_queue.put(line)<EOL><DEDENT>for line in _textio_iterlines(stream):<EOL><INDENT>stream_queue.put(line)<EOL><DEDENT>stream_queue.put(None)  <EOL><DEDENT>stream_queue = queue.Queue(maxsize=buffersize)<EOL>_thread = Thread(target=enqueue_output, args=(proc, stream, stream_queue))<EOL>_thread.daemon = True  <EOL>_thread.start()<EOL>return stream_queue<EOL>

Reads output from a process in a separate thread


if method == '<STR_LIT>':<EOL><INDENT>tz_hour = time.timezone // <NUM_LIT><EOL>utc_offset = str(tz_hour) if tz_hour < <NUM_LIT:0> else '<STR_LIT:+>' + str(tz_hour)<EOL>stamp = time.strftime('<STR_LIT>') + utc_offset<EOL>return stamp<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

make an iso8601 timestamp

Args:
    method (str): type of timestamp

Example:
    >>> stamp = timestamp()
    >>> print('stamp = {!r}'.format(stamp))
    stamp = ...-...-...T...


if key is None and isinstance(indexable, collections_abc.Mapping):<EOL><INDENT>return min(indexable.items(), key=operator.itemgetter(<NUM_LIT:1>))[<NUM_LIT:0>]<EOL><DEDENT>elif hasattr(indexable, '<STR_LIT:index>'):<EOL><INDENT>if key is None:<EOL><INDENT>return indexable.index(min(indexable))<EOL><DEDENT>else:<EOL><INDENT>return indexable.index(min(indexable, key=key))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return argsort(indexable, key=key)[<NUM_LIT:0>]<EOL><DEDENT>

Returns index / key of the item with the smallest value.

This is similar to `numpy.argmin`, but it is written in pure python and
works on both lists and dictionaries.

Args:
    indexable (Iterable or Mapping): indexable to sort by

    key (Callable, optional): customizes the ordering of the indexable

Example:
    >>> assert argmin({'a': 3, 'b': 2, 'c': 100}) == 'b'
    >>> assert argmin(['a', 'c', 'b', 'z', 'f']) == 0
    >>> assert argmin([[0, 1], [2, 3, 4], [5]], key=len) == 2
    >>> assert argmin({'a': 3, 'b': 2, 3: 100, 4: 4}) == 'b'
    >>> assert argmin(iter(['a', 'c', 'A', 'z', 'f'])) == 2


<EOL>if key is None:<EOL><INDENT>return unique(range(len(items)), key=lambda i: items[i])<EOL><DEDENT>else:<EOL><INDENT>return unique(range(len(items)), key=lambda i: key(items[i]))<EOL><DEDENT>

Returns indices corresponding to the first instance of each unique item.

Args:
    items (Sequence): indexable collection of items

    key (Callable, optional): custom normalization function.
        If specified returns items where `key(item)` is unique.

Yields:
    int : indices of the unique items

Example:
    >>> items = [0, 2, 5, 1, 1, 0, 2, 4]
    >>> indices = list(argunique(items))
    >>> assert indices == [0, 1, 2, 3, 7]
    >>> indices = list(argunique(items, key=lambda x: x % 2 == 0))
    >>> assert indices == [0, 2]


return it.chain.from_iterable(nested_list)<EOL>

Transforms a nested iterable into a flat iterable.

This is simply an alias for `itertools.chain.from_iterable`

Args:
    nested_list (Iterable[Iterable]): list of lists

Returns:
    Iterable: flattened items

Example:
    >>> import ubelt as ub
    >>> nested_list = [['a', 'b'], ['c', 'd']]
    >>> list(ub.flatten(nested_list))
    ['a', 'b', 'c', 'd']


if maxval is None:<EOL><INDENT>indices = list(indices)<EOL>maxval = max(indices) + <NUM_LIT:1><EOL><DEDENT>mask = [False] * maxval<EOL>for index in indices:<EOL><INDENT>mask[index] = True<EOL><DEDENT>return mask<EOL>

Constructs a list of booleans where an item is True if its position is in
`indices` otherwise it is False.

Args:
    indices (list): list of integer indices

    maxval (int): length of the returned list. If not specified
        this is inferred from `indices`

Note:
    In the future the arg `maxval` may change its name to `shape`

Returns:
    list: mask: list of booleans. mask[idx] is True if idx in indices

Example:
    >>> import ubelt as ub
    >>> indices = [0, 1, 4]
    >>> mask = ub.boolmask(indices, maxval=6)
    >>> assert mask == [True, True, False, False, True, False]
    >>> mask = ub.boolmask(indices)
    >>> assert mask == [True, True, False, False, True]


if key is None and isinstance(indexable, collections_abc.Mapping):<EOL><INDENT>return max(indexable.items(), key=operator.itemgetter(<NUM_LIT:1>))[<NUM_LIT:0>]<EOL><DEDENT>elif hasattr(indexable, '<STR_LIT:index>'):<EOL><INDENT>if key is None:<EOL><INDENT>return indexable.index(max(indexable))<EOL><DEDENT>else:<EOL><INDENT>return indexable.index(max(indexable, key=key))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return argsort(indexable, key=key)[-<NUM_LIT:1>]<EOL><DEDENT>

Returns index / key of the item with the largest value.

This is similar to `numpy.argmax`, but it is written in pure python and
works on both lists and dictionaries.

Args:
    indexable (Iterable or Mapping): indexable to sort by

    key (Callable, optional): customizes the ordering of the indexable

CommandLine:
    python -m ubelt.util_list argmax

Example:
    >>> assert argmax({'a': 3, 'b': 2, 'c': 100}) == 'c'
    >>> assert argmax(['a', 'c', 'b', 'z', 'f']) == 3
    >>> assert argmax([[0, 1], [2, 3, 4], [5]], key=len) == 1
    >>> assert argmax({'a': 3, 'b': 2, 3: 100, 4: 4}) == 3
    >>> assert argmax(iter(['a', 'c', 'b', 'z', 'f'])) == 3


len_ = len(items)<EOL>if key is None:<EOL><INDENT>item_to_index = dict(zip(reversed(items), reversed(range(len_))))<EOL>indices = item_to_index.values()<EOL><DEDENT>else:<EOL><INDENT>indices = argunique(items, key=key)<EOL><DEDENT>flags = boolmask(indices, len_)<EOL>return flags<EOL>

Returns a list of booleans corresponding to the first instance of each
unique item.

Args:
    items (Sequence): indexable collection of items

    key (Callable, optional): custom normalization function.
        If specified returns items where `key(item)` is unique.

Returns:
    List[bool] : flags the items that are unique

Example:
    >>> import ubelt as ub
    >>> items = [0, 2, 1, 1, 0, 9, 2]
    >>> flags = unique_flags(items)
    >>> assert flags == [True, True, True, False, False, True, False]
    >>> flags = unique_flags(items, key=lambda x: x % 2 == 0)
    >>> assert flags == [True, False, True, False, False, False, False]


return it.compress(items, flags)<EOL>

Selects items where the corresponding value in flags is True
This is similar to np.compress and it.compress

Args:
    items (Iterable): a sequence to select items from

    flags (Iterable): corresponding sequence of bools

Returns:
    Iterable: a subset of masked items

Example:
    >>> import ubelt as ub
    >>> items = [1, 2, 3, 4, 5]
    >>> flags = [False, True, True, False, True]
    >>> list(ub.compress(items, flags))
    [2, 3, 5]


return (items[index] for index in indices)<EOL>

Selects a subset of a list based on a list of indices.
This is similar to np.take, but pure python.

Args:
    items (Sequence): an indexable object to select items from

    indices (Iterable): sequence of indexing objects

Returns:
    Iterable or scalar: subset of the list

SeeAlso:
    ub.dict_subset

Example:
    >>> import ubelt as ub
    >>> items = [0, 1, 2, 3]
    >>> indices = [2, 0]
    >>> list(ub.take(items, indices))
    [2, 0]


if not jwfs.is_reparse_point(path):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>handle = jwfs.api.CreateFile(<EOL>path, <NUM_LIT:0>, <NUM_LIT:0>, None, jwfs.api.OPEN_EXISTING,<EOL>jwfs.api.FILE_FLAG_OPEN_REPARSE_POINT |<EOL>jwfs.api.FILE_FLAG_BACKUP_SEMANTICS,<EOL>None)<EOL>if handle == jwfs.api.INVALID_HANDLE_VALUE:<EOL><INDENT>raise WindowsError()<EOL><DEDENT>res = jwfs.reparse.DeviceIoControl(<EOL>handle, jwfs.api.FSCTL_GET_REPARSE_POINT, None, <NUM_LIT>)<EOL>bytes = jwfs.create_string_buffer(res)<EOL>p_rdb = jwfs.cast(bytes, jwfs.POINTER(jwfs.api.REPARSE_DATA_BUFFER))<EOL>rdb = p_rdb.contents<EOL>if rdb.tag not in [<NUM_LIT>, jwfs.api.IO_REPARSE_TAG_SYMLINK]:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>" % rdb.tag)<EOL><DEDENT>jwfs.handle_nonzero_success(jwfs.api.CloseHandle(handle))<EOL>subname = rdb.get_substitute_name()<EOL>if subname.startswith('<STR_LIT>'):<EOL><INDENT>subname = subname[<NUM_LIT:2>:]<EOL><DEDENT>return subname<EOL>

Returns the location that the junction points, raises ValueError if path is
not a junction.

CommandLine:
    python -m ubelt._win32_links _win32_read_junction

Example:
    >>> # xdoc: +REQUIRES(WIN32)
    >>> import ubelt as ub
    >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')
    >>> ub.delete(root)
    >>> ub.ensuredir(root)
    >>> dpath = join(root, 'dpath')
    >>> djunc = join(root, 'djunc')
    >>> ub.ensuredir(dpath)
    >>> _win32_junction(dpath, djunc)
    >>> path = djunc
    >>> pointed = _win32_read_junction(path)
    >>> print('pointed = {!r}'.format(pointed))


from ubelt import util_cmd<EOL>import re<EOL>wrapper = '<STR_LIT>'  <EOL>command = '<STR_LIT>'.format(path, star)<EOL>wrapped = wrapper.format(command)<EOL>info = util_cmd.cmd(wrapped, shell=True)<EOL>if info['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>from ubelt import util_format<EOL>print('<STR_LIT>')<EOL>print(info['<STR_LIT>'])<EOL>print(util_format.repr2(info, nl=<NUM_LIT:1>))<EOL>raise OSError(str(info))<EOL><DEDENT>lines = info['<STR_LIT>'].split('<STR_LIT:\n>')[<NUM_LIT:5>:-<NUM_LIT:3>]<EOL>splitter = re.compile('<STR_LIT>')<EOL>for line in lines:<EOL><INDENT>parts = splitter.split(line)<EOL>date, sep, time, sep, ampm, sep, type_or_size, sep = parts[:<NUM_LIT:8>]<EOL>name = '<STR_LIT>'.join(parts[<NUM_LIT:8>:])<EOL>if name == '<STR_LIT:.>' or name == '<STR_LIT:..>':<EOL><INDENT>continue<EOL><DEDENT>if type_or_size in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>pos = name.find('<STR_LIT::>')<EOL>bpos = name[:pos].rfind('<STR_LIT:[>')<EOL>name = name[:bpos - <NUM_LIT:1>]<EOL>pointed = name[bpos + <NUM_LIT:1>:-<NUM_LIT:1>]<EOL>yield type_or_size, name, pointed<EOL><DEDENT>else:<EOL><INDENT>yield type_or_size, name, None<EOL><DEDENT><DEDENT>

Using the windows cmd shell to get information about a directory


<EOL>def _rmjunctions(root):<EOL><INDENT>subdirs = []<EOL>for name in os.listdir(root):<EOL><INDENT>current = join(root, name)<EOL>if os.path.isdir(current):<EOL><INDENT>if _win32_is_junction(current):<EOL><INDENT>os.rmdir(current)<EOL><DEDENT>elif not os.path.islink(current):<EOL><INDENT>subdirs.append(current)<EOL><DEDENT><DEDENT><DEDENT>for subdir in subdirs:<EOL><INDENT>_rmjunctions(subdir)<EOL><DEDENT><DEDENT>if _win32_is_junction(path):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.rmdir(path)<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>_rmjunctions(path)<EOL>import shutil<EOL>shutil.rmtree(path)<EOL><DEDENT>

rmtree for win32 that treats junctions like directory symlinks.
The junction removal portion may not be safe on race conditions.

There is a known issue that prevents shutil.rmtree from
deleting directories with junctions.
https://bugs.python.org/issue31226


if argv is None:  <EOL><INDENT>argv = sys.argv<EOL><DEDENT>keys = [key] if isinstance(key, six.string_types) else key<EOL>flag = any(k in argv for k in keys)<EOL>return flag<EOL>

Determines if a key is specified on the command line

Args:
    key (str or tuple): string or tuple of strings. Each key should be
        prefixed with two hyphens (i.e. `--`)
    argv (Optional[list]): overrides `sys.argv` if specified

Returns:
    bool: flag : True if the key (or any of the keys) was specified

Example:
    >>> import ubelt as ub
    >>> argv = ['--spam', '--eggs', 'foo']
    >>> assert ub.argflag('--eggs', argv=argv) is True
    >>> assert ub.argflag('--ans', argv=argv) is False
    >>> assert ub.argflag('foo', argv=argv) is True
    >>> assert ub.argflag(('bar', '--spam'), argv=argv) is True


def argval(key, default=util_const.NoParam, argv=None):

if argv is None:  <EOL><INDENT>argv = sys.argv<EOL><DEDENT>keys = [key] if isinstance(key, six.string_types) else key<EOL>n_max = len(argv) - <NUM_LIT:1><EOL>for argx, item in enumerate(argv):<EOL><INDENT>for key_ in keys:<EOL><INDENT>if item == key_:<EOL><INDENT>if argx < n_max:<EOL><INDENT>value = argv[argx + <NUM_LIT:1>]<EOL>return value<EOL><DEDENT><DEDENT>elif item.startswith(key_ + '<STR_LIT:=>'):<EOL><INDENT>value = '<STR_LIT:=>'.join(item.split('<STR_LIT:=>')[<NUM_LIT:1>:])<EOL>return value<EOL><DEDENT><DEDENT><DEDENT>value = default<EOL>return value<EOL>

Get the value of a keyword argument specified on the command line.

Values can be specified as `<key> <value>` or `<key>=<value>`

Args:
    key (str or tuple): string or tuple of strings. Each key should be
        prefixed with two hyphens (i.e. `--`)
    default (Optional[object]): value to return if not specified
    argv (Optional[list]): overrides `sys.argv` if specified

Returns:
    str: value : the value specified after the key. It they key is
        specified multiple times, then the first value is returned.

TODO:
    - [ ] Can we handle the case where the value is a list of long paths?
    - [ ] Should we default the first or last specified instance of the flag.

Example:
    >>> import ubelt as ub
    >>> argv = ['--ans', '42', '--quest=the grail', '--ans=6', '--bad']
    >>> assert ub.argval('--spam', argv=argv) == ub.NoParam
    >>> assert ub.argval('--quest', argv=argv) == 'the grail'
    >>> assert ub.argval('--ans', argv=argv) == '42'
    >>> assert ub.argval('--bad', argv=argv) == ub.NoParam
    >>> assert ub.argval(('--bad', '--bar'), argv=argv) == ub.NoParam

Example:
    >>> # Test fix for GH Issue #41
    >>> import ubelt as ub
    >>> argv = ['--path=/path/with/k=3']
    >>> ub.argval('--path', argv=argv) == '/path/with/k=3'


def touch(fpath, mode=<NUM_LIT>, dir_fd=None, verbose=<NUM_LIT:0>, **kwargs):

if verbose:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>if six.PY2:  <EOL><INDENT>with open(fpath, '<STR_LIT:a>'):<EOL><INDENT>os.utime(fpath, None)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>flags = os.O_CREAT | os.O_APPEND<EOL>with os.fdopen(os.open(fpath, flags=flags, mode=mode, dir_fd=dir_fd)) as f:<EOL><INDENT>os.utime(f.fileno() if os.utime in os.supports_fd else fpath,<EOL>dir_fd=None if os.supports_fd else dir_fd, **kwargs)<EOL><DEDENT><DEDENT>return fpath<EOL>

change file timestamps

Works like the touch unix utility

Args:
    fpath (PathLike): name of the file
    mode (int): file permissions (python3 and unix only)
    dir_fd (file): optional directory file descriptor. If specified, fpath
        is interpreted as relative to this descriptor (python 3 only).
    verbose (int): verbosity
    **kwargs : extra args passed to `os.utime` (python 3 only).

Returns:
    PathLike: path to the file

References:
    https://stackoverflow.com/questions/1158076/implement-touch-using-python

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_cache_dir('ubelt')
    >>> fpath = join(dpath, 'touch_file')
    >>> assert not exists(fpath)
    >>> ub.touch(fpath)
    >>> assert exists(fpath)
    >>> os.unlink(fpath)


def writeto(fpath, to_write, aslines=False, verbose=None):

if verbose:<EOL><INDENT>print('<STR_LIT>' % (fpath,))<EOL><DEDENT>with open(fpath, '<STR_LIT:wb>') as file:<EOL><INDENT>if aslines:<EOL><INDENT>to_write = map(_ensure_bytes , to_write)<EOL>file.writelines(to_write)<EOL><DEDENT>else:<EOL><INDENT>bytes = _ensure_bytes(to_write)<EOL>file.write(bytes)<EOL><DEDENT><DEDENT>

r"""
    Writes (utf8) text to a file.

    Args:
        fpath (PathLike): file path
        to_write (str): text to write (must be unicode text)
        aslines (bool): if True to_write is assumed to be a list of lines
        verbose (bool): verbosity flag

    CommandLine:
        python -m ubelt.util_io writeto --verbose

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt')
        >>> fpath = dpath + '/' + 'testwrite.txt'
        >>> if exists(fpath):
        >>>     os.remove(fpath)
        >>> to_write = 'utf-8 symbols Δ, Й, ק, م, ๗, あ, 叶, 葉, and 말.'
        >>> writeto(fpath, to_write)
        >>> read_ = ub.readfrom(fpath)
        >>> print('read_    = ' + read_)
        >>> print('to_write = ' + to_write)
        >>> assert read_ == to_write

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt')
        >>> fpath = dpath + '/' + 'testwrite2.txt'
        >>> if exists(fpath):
        >>>     os.remove(fpath)
        >>> to_write = ['a\n', 'b\n', 'c\n', 'd\n']
        >>> writeto(fpath, to_write, aslines=True)
        >>> read_ = ub.readfrom(fpath, aslines=True)
        >>> print('read_    = {}'.format(read_))
        >>> print('to_write = {}'.format(to_write))
        >>> assert read_ == to_write


def readfrom(fpath, aslines=False, errors='<STR_LIT:replace>', verbose=None):

if verbose:<EOL><INDENT>print('<STR_LIT>' % (fpath,))<EOL><DEDENT>if not exists(fpath):<EOL><INDENT>raise IOError('<STR_LIT>' % (fpath,))<EOL><DEDENT>with open(fpath, '<STR_LIT:rb>') as file:<EOL><INDENT>if aslines:<EOL><INDENT>text = [line.decode('<STR_LIT:utf8>', errors=errors)<EOL>for line in file.readlines()]<EOL>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>text = [<EOL>line[:-<NUM_LIT:2>] + '<STR_LIT:\n>' if line.endswith('<STR_LIT:\r\n>') else line<EOL>for line in text<EOL>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>text = file.read().decode('<STR_LIT:utf8>', errors=errors)<EOL><DEDENT><DEDENT>return text<EOL>

Reads (utf8) text from a file.

Args:
    fpath (PathLike): file path
    aslines (bool): if True returns list of lines
    verbose (bool): verbosity flag

Returns:
    str: text from fpath (this is unicode)


def find_duplicates(items, k=<NUM_LIT:2>, key=None):

<EOL>duplicates = defaultdict(list)<EOL>if key is None:<EOL><INDENT>for count, item in enumerate(items):<EOL><INDENT>duplicates[item].append(count)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for count, item in enumerate(items):<EOL><INDENT>duplicates[key(item)].append(count)<EOL><DEDENT><DEDENT>for key in list(duplicates.keys()):<EOL><INDENT>if len(duplicates[key]) < k:<EOL><INDENT>del duplicates[key]<EOL><DEDENT><DEDENT>duplicates = dict(duplicates)<EOL>return duplicates<EOL>

Find all duplicate items in a list.

Search for all items that appear more than `k` times and return a mapping
from each (k)-duplicate item to the positions it appeared in.

Args:
    items (Iterable): hashable items possibly containing duplicates
    k (int): only return items that appear at least `k` times (default=2)
    key (Callable, optional): Returns indices where `key(items[i])`
        maps to a particular value at least k times.

Returns:
    dict: maps each duplicate item to the indices at which it appears

CommandLine:
    python -m ubelt.util_dict find_duplicates

Example:
    >>> import ubelt as ub
    >>> items = [0, 0, 1, 2, 3, 3, 0, 12, 2, 9]
    >>> duplicates = ub.find_duplicates(items)
    >>> print('items = %r' % (items,))
    >>> print('duplicates = %r' % (duplicates,))
    >>> assert duplicates == {0: [0, 1, 6], 2: [3, 8], 3: [4, 5]}
    >>> assert ub.find_duplicates(items, 3) == {0: [0, 1, 6]}

Example:
    >>> import ubelt as ub
    >>> items = [0, 0, 1, 2, 3, 3, 0, 12, 2, 9]
    >>> # note: k can be 0
    >>> duplicates = ub.find_duplicates(items, k=0)
    >>> print(ub.repr2(duplicates, nl=0))
    {0: [0, 1, 6], 1: [2], 2: [3, 8], 3: [4, 5], 9: [9], 12: [7]}

Example:
    >>> import ubelt as ub
    >>> items = [10, 11, 12, 13, 14, 15, 16]
    >>> duplicates = ub.find_duplicates(items, key=lambda x: x // 2)
    >>> print(ub.repr2(duplicates, nl=0))
    {5: [0, 1], 6: [2, 3], 7: [4, 5]}


if unique_vals:<EOL><INDENT>if isinstance(dict_, OrderedDict):<EOL><INDENT>inverted = OrderedDict((val, key) for key, val in dict_.items())<EOL><DEDENT>else:<EOL><INDENT>inverted = {val: key for key, val in dict_.items()}<EOL><DEDENT><DEDENT>else:<EOL><INDENT>inverted = defaultdict(set)<EOL>for key, value in dict_.items():<EOL><INDENT>inverted[value].add(key)<EOL><DEDENT>inverted = dict(inverted)<EOL><DEDENT>return inverted<EOL>

r"""
    Swaps the keys and values in a dictionary.

    Args:
        dict_ (dict): dictionary to invert
        unique_vals (bool): if False, inverted keys are returned in a set.
            The default is True.

    Returns:
        dict: inverted

    Notes:
        The must values be hashable.

        If the original dictionary contains duplicate values, then only one of
        the corresponding keys will be returned and the others will be
        discarded.  This can be prevented by setting `unique_vals=True`,
        causing the inverted keys to be returned in a set.

    CommandLine:
        python -m ubelt.util_dict invert_dict

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': 1, 'b': 2}
        >>> inverted = ub.invert_dict(dict_)
        >>> assert inverted == {1: 'a', 2: 'b'}

    Example:
        >>> import ubelt as ub
        >>> dict_ = ub.odict([(2, 'a'), (1, 'b'), (0, 'c'), (None, 'd')])
        >>> inverted = ub.invert_dict(dict_)
        >>> assert list(inverted.keys())[0] == 'a'

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': 1, 'b': 0, 'c': 0, 'd': 0, 'f': 2}
        >>> inverted = ub.invert_dict(dict_, unique_vals=False)
        >>> assert inverted == {0: {'b', 'c', 'd'}, 1: {'a'}, 2: {'f'}}


try:<EOL><INDENT>len(items1)<EOL><DEDENT>except TypeError:<EOL><INDENT>items1 = list(items1)<EOL><DEDENT>try:<EOL><INDENT>len(items2)<EOL><DEDENT>except TypeError:<EOL><INDENT>items2 = list(items2)<EOL><DEDENT>if len(items1) == <NUM_LIT:0> and len(items2) == <NUM_LIT:1>:<EOL><INDENT>items2 = []<EOL><DEDENT>if len(items2) == <NUM_LIT:1> and len(items1) > <NUM_LIT:1>:<EOL><INDENT>items2 = items2 * len(items1)<EOL><DEDENT>if len(items1) != len(items2):<EOL><INDENT>raise ValueError('<STR_LIT>' % (<EOL>len(items1), len(items2)))<EOL><DEDENT>return cls(zip(items1, items2))<EOL>

Zips elementwise pairs between items1 and items2 into a dictionary. Values
from items2 can be broadcast onto items1.

Args:
    items1 (Iterable): full sequence
    items2 (Iterable): can either be a sequence of one item or a sequence
        of equal length to `items1`
    cls (Type[dict]): dictionary type to use. Defaults to dict, but could
        be ordered dict instead.

Returns:
    dict: similar to dict(zip(items1, items2))

Example:
    >>> assert dzip([1, 2, 3], [4]) == {1: 4, 2: 4, 3: 4}
    >>> assert dzip([1, 2, 3], [4, 4, 4]) == {1: 4, 2: 4, 3: 4}
    >>> assert dzip([], [4]) == {}


if not args:<EOL><INDENT>return {}<EOL><DEDENT>else:<EOL><INDENT>dictclass = OrderedDict if isinstance(args[<NUM_LIT:0>], OrderedDict) else dict<EOL>common_keys = set.intersection(*map(set, args))<EOL>first_dict = args[<NUM_LIT:0>]<EOL>return dictclass((k, first_dict[k]) for k in common_keys)<EOL><DEDENT>

Constructs a dictionary that contains keys common between all inputs.
The returned values will only belong to the first dictionary.

Args:
    *args : a sequence of dictionaries (or sets of keys)

Returns:
    Dict | OrderedDict :
        OrderedDict if the first argument is an OrderedDict, otherwise dict

Notes:
    This function can be used as an alternative to `dict_subset` where any
    key not in the dictionary is ignored. See the following example:

    >>> dict_isect({'a': 1, 'b': 2, 'c': 3}, ['a', 'c', 'd'])
    {'a': 1, 'c': 3}

Example:
    >>> dict_isect({'a': 1, 'b': 1}, {'b': 2, 'c': 2})
    {'b': 1}
    >>> dict_isect(odict([('a', 1), ('b', 2)]), odict([('c', 3)]))
    OrderedDict()
    >>> dict_isect()
    {}


def dict_take(dict_, keys, default=util_const.NoParam):

if default is util_const.NoParam:<EOL><INDENT>for key in keys:<EOL><INDENT>yield dict_[key]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for key in keys:<EOL><INDENT>yield dict_.get(key, default)<EOL><DEDENT><DEDENT>

r"""
    Generates values from a dictionary

    Args:
        dict_ (Mapping): a dictionary to take from
        keys (Iterable): the keys to take
        default (object, optional): if specified uses default if keys are missing

    CommandLine:
        python -m ubelt.util_dict dict_take_gen

    Example:
        >>> import ubelt as ub
        >>> dict_ = {1: 'a', 2: 'b', 3: 'c'}
        >>> keys = [1, 2, 3, 4, 5]
        >>> result = list(ub.dict_take(dict_, keys, None))
        >>> assert result == ['a', 'b', 'c', None, None]

    Example:
        >>> import ubelt as ub
        >>> dict_ = {1: 'a', 2: 'b', 3: 'c'}
        >>> keys = [1, 2, 3, 4, 5]
        >>> try:
        >>>     print(list(ub.dict_take(dict_, keys)))
        >>>     raise AssertionError('did not get key error')
        >>> except KeyError:
        >>>     print('correctly got key error')


return self._base(<EOL>(key, (value.to_dict() if isinstance(value, AutoDict) else value))<EOL>for key, value in self.items())<EOL>

Recursively casts a AutoDict into a regular dictionary. All nested
AutoDict values are also converted.

Returns:
    dict: a copy of this dict without autovivification

Example:
    >>> from ubelt.util_dict import AutoDict
    >>> auto = AutoDict()
    >>> auto[1] = 1
    >>> auto['n1'] = AutoDict()
    >>> static = auto.to_dict()
    >>> assert not isinstance(static, AutoDict)
    >>> assert not isinstance(static['n1'], AutoDict)


if callable(groupids):<EOL><INDENT>keyfunc = groupids<EOL>pair_list = ((keyfunc(item), item) for item in items)<EOL><DEDENT>else:<EOL><INDENT>pair_list = zip(groupids, items)<EOL><DEDENT>groupid_to_items = defaultdict(list)<EOL>for key, item in pair_list:<EOL><INDENT>groupid_to_items[key].append(item)<EOL><DEDENT>return groupid_to_items<EOL>

r"""
    Groups a list of items by group id.

    Args:
        items (Iterable): a list of items to group
        groupids (Iterable or Callable): a corresponding list of item groupids
            or a function mapping an item to a groupid.

    Returns:
        dict: groupid_to_items: maps a groupid to a list of items

    CommandLine:
        python -m ubelt.util_dict group_items

    Example:
        >>> import ubelt as ub
        >>> items    = ['ham',     'jam',   'spam',     'eggs',    'cheese', 'banana']
        >>> groupids = ['protein', 'fruit', 'protein',  'protein', 'dairy',  'fruit']
        >>> groupid_to_items = ub.group_items(items, groupids)
        >>> print(ub.repr2(groupid_to_items, nl=0))
        {'dairy': ['cheese'], 'fruit': ['jam', 'banana'], 'protein': ['ham', 'spam', 'eggs']}


if LINUX:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>elif DARWIN:  <EOL><INDENT>dpath_  = '<STR_LIT>'<EOL><DEDENT>elif WIN32:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:  <EOL><INDENT>raise '<STR_LIT>'<EOL><DEDENT>dpath = normpath(expanduser(dpath_))<EOL>return dpath<EOL>

Returns path for user-specific data files

Returns:
    PathLike : path to the data dir used by the current operating system


from six import types<EOL>from ubelt import util_cmd<EOL>import warnings<EOL>warnings.warn('<STR_LIT>', DeprecationWarning)<EOL>if not isinstance(fpath, six.string_types):<EOL><INDENT>if isinstance(fpath, types.ModuleType):<EOL><INDENT>fpath = fpath.__file__<EOL><DEDENT>else:<EOL><INDENT>fpath =  sys.modules[fpath.__module__].__file__<EOL><DEDENT>fpath_py = fpath.replace('<STR_LIT>', '<STR_LIT>')<EOL>if exists(fpath_py):<EOL><INDENT>fpath = fpath_py<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>editor = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL>if not exists(fpath):<EOL><INDENT>raise IOError('<STR_LIT>' % fpath)<EOL><DEDENT>util_cmd.cmd([editor, fpath], fpath, detach=True)<EOL>

DEPRICATED: This has been ported to xdev, please use that version.

Opens a file or code corresponding to a live python object in your
preferred visual editor. This function is mainly useful in an interactive
IPython session.

The visual editor is determined by the `VISUAL` environment variable.  If
this is not specified it defaults to gvim.

Args:
    fpath (PathLike): a file path or python module / function
    verbose (int): verbosity

DisableExample:
    >>> # This test interacts with a GUI frontend, not sure how to test.
    >>> import ubelt as ub
    >>> ub.editfile(ub.util_platform.__file__)
    >>> ub.editfile(ub)
    >>> ub.editfile(ub.editfile)


dpath = join(platform_config_dir(), appname, *args)<EOL>return dpath<EOL>

r"""
    Returns a writable directory for an application
    This should be used for persistent configuration files.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable config directory for this application

    SeeAlso:
        ensure_app_config_dir


from ubelt import util_path<EOL>dpath = get_app_config_dir(appname, *args)<EOL>util_path.ensuredir(dpath)<EOL>return dpath<EOL>

Calls `get_app_config_dir` but ensures the directory exists.

Args:
    appname (str): the name of the application
    *args: any other subdirectories may be specified

SeeAlso:
    get_app_config_dir

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_config_dir('ubelt')
    >>> assert exists(dpath)


dpath = join(platform_cache_dir(), appname, *args)<EOL>return dpath<EOL>

r"""
    Returns a writable directory for an application.
    This should be used for temporary deletable data.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable cache directory for this application

    SeeAlso:
        ensure_app_cache_dir


if LINUX:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>elif DARWIN:  <EOL><INDENT>dpath_  = '<STR_LIT>'<EOL><DEDENT>elif WIN32:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:  <EOL><INDENT>raise NotImplementedError('<STR_LIT>' % (sys.platform,))<EOL><DEDENT>dpath = normpath(expanduser(dpath_))<EOL>return dpath<EOL>

Returns a directory which should be writable for any application
This should be used for persistent configuration files.

Returns:
    PathLike : path to the cahce dir used by the current operating system


dpath = join(platform_data_dir(), appname, *args)<EOL>return dpath<EOL>

r"""
    Returns a writable directory for an application.
    This should be used for temporary deletable data.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable data directory for this application

    SeeAlso:
        ensure_app_data_dir


return ensure_app_cache_dir(appname, *args)<EOL>

Calls `get_app_resource_dir` but ensures the directory exists.

DEPRICATED in favor of ensure_app_config_dir / ensure_app_data_dir

Args:
    appname (str): the name of the application
    *args: any other subdirectories may be specified

SeeAlso:
    get_app_resource_dir


Returns a directory which should be writable for any application
This should be used for temporary deletable data.

Returns:
    PathLike : path to the cache dir used by the current operating system


from ubelt import util_cmd<EOL>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>fpath = normpath(fpath)<EOL>if not exists(fpath):<EOL><INDENT>raise Exception('<STR_LIT>' % fpath)<EOL><DEDENT>if not WIN32:<EOL><INDENT>import pipes<EOL>fpath = pipes.quote(fpath)<EOL><DEDENT>if LINUX:<EOL><INDENT>info = util_cmd.cmd(('<STR_LIT>', fpath), detach=True, verbose=verbose)<EOL><DEDENT>elif DARWIN:<EOL><INDENT>info = util_cmd.cmd(('<STR_LIT>', fpath), detach=True, verbose=verbose)<EOL><DEDENT>elif WIN32:<EOL><INDENT>os.startfile(fpath)<EOL>info = None<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>if info is not None:<EOL><INDENT>if not info['<STR_LIT>']:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT><DEDENT>

Uses default program defined by the system to open a file.
This is done via `os.startfile` on windows, `open` on mac, and `xdg-open`
on linux.

Args:
    fpath (PathLike): a file to open using the program associated with the
        files extension type.
    verbose (int): verbosity

References:
    http://stackoverflow.com/questions/2692873/quote-posix

DisableExample:
    >>> # This test interacts with a GUI frontend, not sure how to test.
    >>> import ubelt as ub
    >>> base = ub.ensure_app_cache_dir('ubelt')
    >>> fpath1 = join(base, 'test_open.txt')
    >>> ub.touch(fpath1)
    >>> proc = ub.startfile(fpath1)


def inject_method(self, func, name=None):

<EOL>new_method = func.__get__(self, self.__class__)<EOL>if name is None:<EOL><INDENT>name = func.__name__<EOL><DEDENT>setattr(self, name, new_method)<EOL>

Injects a function into an object instance as a bound method

The main use case of this function is for monkey patching. While monkey
patching is sometimes necessary it should generally be avoided. Thus, we
simply remind the developer that there might be a better way.

Args:
    self (object): instance to inject a function into
    func (func): the function to inject (must contain an arg for self)
    name (str): name of the method. optional. If not specified the name
        of the function is used.

Example:
    >>> class Foo(object):
    >>>     def bar(self):
    >>>         return 'bar'
    >>> def baz(self):
    >>>     return 'baz'
    >>> self = Foo()
    >>> assert self.bar() == 'bar'
    >>> assert not hasattr(self, 'baz')
    >>> inject_method(self, baz)
    >>> assert not hasattr(Foo, 'baz'), 'should only change one instance'
    >>> assert self.baz() == 'baz'
    >>> inject_method(self, baz, 'bar')
    >>> assert self.bar() == 'baz'


from six.moves import cPickle as pickle<EOL>if not self.enabled:<EOL><INDENT>return<EOL><DEDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT>cfgstr = self._rectify_cfgstr(cfgstr)<EOL>condensed = self._condense_cfgstr(cfgstr)<EOL>util_path.ensuredir(self.dpath)<EOL>data_fpath = self.get_fpath(cfgstr=cfgstr)<EOL>meta_fpath = data_fpath + '<STR_LIT>'<EOL>with open(meta_fpath, '<STR_LIT:a>') as file_:<EOL><INDENT>file_.write('<STR_LIT>'.format(util_time.timestamp()))<EOL>file_.write(self.fname + '<STR_LIT:\n>')<EOL>file_.write(condensed + '<STR_LIT:\n>')<EOL>file_.write(cfgstr + '<STR_LIT:\n>')<EOL>file_.write(str(self.meta) + '<STR_LIT:\n>')<EOL><DEDENT>with open(data_fpath, '<STR_LIT:wb>') as file_:<EOL><INDENT>pickle.dump(data, file_, protocol=self.protocol)<EOL><DEDENT>

Writes data to path specified by `self.fpath(cfgstr)`.

Metadata containing information about the cache will also be appended
to an adjacent file with the `.meta` suffix.

Example:
    >>> from ubelt.util_cache import *  # NOQA
    >>> # Normal functioning
    >>> cfgstr = 'long-cfg' * 32
    >>> cacher = Cacher('test_enabled_save', cfgstr)
    >>> cacher.save('data')
    >>> assert exists(cacher.get_fpath()), 'should be enabeled'
    >>> assert exists(cacher.get_fpath() + '.meta'), 'missing metadata'
    >>> # Setting the cacher as enabled=False turns it off
    >>> cacher2 = Cacher('test_disabled_save', 'params', enabled=False)
    >>> cacher2.save('data')
    >>> assert not exists(cacher2.get_fpath()), 'should be disabled'


return exists(self.get_fpath())<EOL>


condensed = self._condense_cfgstr(cfgstr)<EOL>fname_cfgstr = '<STR_LIT>'.format(self.fname, condensed, self.ext)<EOL>fpath = join(self.dpath, fname_cfgstr)<EOL>fpath = normpath(fpath)<EOL>return fpath<EOL>

Reports the filepath that the cacher will use.
It will attempt to use '{fname}_{cfgstr}{ext}' unless that is too long.
Then cfgstr will be hashed.

Example:
    >>> from ubelt.util_cache import Cacher
    >>> import pytest
    >>> with pytest.warns(UserWarning):
    >>>     cacher = Cacher('test_cacher1')
    >>>     cacher.get_fpath()
    >>> self = Cacher('test_cacher2', cfgstr='cfg1')
    >>> self.get_fpath()
    >>> self = Cacher('test_cacher3', cfgstr='cfg1' * 32)
    >>> self.get_fpath()


import glob<EOL>pattern = join(self.dpath, self.fname + '<STR_LIT>' + self.ext)<EOL>for fname in glob.iglob(pattern):<EOL><INDENT>data_fpath = join(self.dpath, fname)<EOL>yield data_fpath<EOL><DEDENT>

Returns data with different cfgstr values that were previously computed
with this cacher.

Example:
    >>> from ubelt.util_cache import Cacher
    >>> # Ensure that some data exists
    >>> known_fnames = set()
    >>> cacher = Cacher('versioned_data', cfgstr='1')
    >>> cacher.ensure(lambda: 'data1')
    >>> known_fnames.add(cacher.get_fpath())
    >>> cacher = Cacher('versioned_data', cfgstr='2')
    >>> cacher.ensure(lambda: 'data2')
    >>> known_fnames.add(cacher.get_fpath())
    >>> # List previously computed configs for this type
    >>> from os.path import basename
    >>> cacher = Cacher('versioned_data', cfgstr='2')
    >>> exist_fpaths = set(cacher.existing_versions())
    >>> exist_fnames = list(map(basename, exist_fpaths))
    >>> print(exist_fnames)
    >>> assert exist_fpaths == known_fnames

    ['versioned_data_1.pkl', 'versioned_data_2.pkl']


def tryload(self, cfgstr=None, on_error='<STR_LIT>'):

cfgstr = self._rectify_cfgstr(cfgstr)<EOL>if self.enabled:<EOL><INDENT>try:<EOL><INDENT>if self.verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT>return self.load(cfgstr)<EOL><DEDENT>except IOError:<EOL><INDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>')<EOL><DEDENT>if on_error == '<STR_LIT>':<EOL><INDENT>raise<EOL><DEDENT>elif on_error == '<STR_LIT>':<EOL><INDENT>self.clear(cfgstr)<EOL>return None<EOL><DEDENT>else:<EOL><INDENT>raise KeyError('<STR_LIT>'.format(on_error))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if self.verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT><DEDENT>return None<EOL>

Like load, but returns None if the load fails due to a cache miss.

Args:
    on_error (str): How to handle non-io errors errors. Either raise,
        which re-raises the exception, or clear which deletes the cache
        and returns None.


def ensure(self, func, *args, **kwargs):

data = self.tryload()<EOL>if data is None:<EOL><INDENT>data = func(*args, **kwargs)<EOL>self.save(data)<EOL><DEDENT>return data<EOL>

r"""
        Wraps around a function. A cfgstr must be stored in the base cacher.

        Args:
            func (callable): function that will compute data on cache miss
            *args: passed to func
            **kwargs: passed to func

        Example:
            >>> from ubelt.util_cache import *  # NOQA
            >>> def func():
            >>>     return 'expensive result'
            >>> fname = 'test_cacher_ensure'
            >>> cfgstr = 'func params'
            >>> cacher = Cacher(fname, cfgstr)
            >>> cacher.clear()
            >>> data1 = cacher.ensure(func)
            >>> data2 = cacher.ensure(func)
            >>> assert data1 == 'expensive result'
            >>> assert data1 == data2
            >>> cacher.clear()


for e in es:<EOL><INDENT>ensure_freshness(e)<EOL><DEDENT>for (fb,bb) in self.builder_layers:<EOL><INDENT>fs = fb.initial_state().transduce(es)<EOL>bs = bb.initial_state().transduce(reversed(es))<EOL>es = [concatenate([f,b]) for f,b in zip(fs, reversed(bs))]<EOL><DEDENT>return es<EOL>

returns the list of output Expressions obtained by adding the given inputs
to the current state, one by one, to both the forward and backward RNNs,
and concatenating.

@param es: a list of Expression

see also add_inputs(xs)

.transduce(xs) is different from .add_inputs(xs) in the following way:

    .add_inputs(xs) returns a list of RNNState pairs. RNNState objects can be
     queried in various ways. In particular, they allow access to the previous
     state, as well as to the state-vectors (h() and s() )

    .transduce(xs) returns a list of Expression. These are just the output
     expressions. For many cases, this suffices.
     transduce is much more memory efficient than add_inputs.