len_ = len(items)<EOL>if key is None:<EOL><INDENT>item_to_index = dict(zip(reversed(items), reversed(range(len_))))<EOL>indices = item_to_index.values()<EOL><DEDENT>else:<EOL><INDENT>indices = argunique(items, key=key)<EOL><DEDENT>flags = boolmask(indices, len_)<EOL>return flags<EOL>

Returns a list of booleans corresponding to the first instance of each
unique item.

Args:
    items (Sequence): indexable collection of items

    key (Callable, optional): custom normalization function.
        If specified returns items where `key(item)` is unique.

Returns:
    List[bool] : flags the items that are unique

Example:
    >>> import ubelt as ub
    >>> items = [0, 2, 1, 1, 0, 9, 2]
    >>> flags = unique_flags(items)
    >>> assert flags == [True, True, True, False, False, True, False]
    >>> flags = unique_flags(items, key=lambda x: x % 2 == 0)
    >>> assert flags == [True, False, True, False, False, False, False]


if maxval is None:<EOL><INDENT>indices = list(indices)<EOL>maxval = max(indices) + <NUM_LIT:1><EOL><DEDENT>mask = [False] * maxval<EOL>for index in indices:<EOL><INDENT>mask[index] = True<EOL><DEDENT>return mask<EOL>

Constructs a list of booleans where an item is True if its position is in
`indices` otherwise it is False.

Args:
    indices (list): list of integer indices

    maxval (int): length of the returned list. If not specified
        this is inferred from `indices`

Note:
    In the future the arg `maxval` may change its name to `shape`

Returns:
    list: mask: list of booleans. mask[idx] is True if idx in indices

Example:
    >>> import ubelt as ub
    >>> indices = [0, 1, 4]
    >>> mask = ub.boolmask(indices, maxval=6)
    >>> assert mask == [True, True, False, False, True, False]
    >>> mask = ub.boolmask(indices)
    >>> assert mask == [True, True, False, False, True]


def iter_window(iterable, size=<NUM_LIT:2>, step=<NUM_LIT:1>, wrap=False):

<EOL>iter_list = it.tee(iterable, size)<EOL>if wrap:<EOL><INDENT>iter_list = [iter_list[<NUM_LIT:0>]] + list(map(it.cycle, iter_list[<NUM_LIT:1>:]))<EOL><DEDENT>try:<EOL><INDENT>for count, iter_ in enumerate(iter_list[<NUM_LIT:1>:], start=<NUM_LIT:1>):<EOL><INDENT>for _ in range(count):<EOL><INDENT>next(iter_)<EOL><DEDENT><DEDENT><DEDENT>except StopIteration:<EOL><INDENT>return iter(())<EOL><DEDENT>else:<EOL><INDENT>_window_iter = zip(*iter_list)<EOL>window_iter = it.islice(_window_iter, <NUM_LIT:0>, None, step)<EOL>return window_iter<EOL><DEDENT>

Iterates through iterable with a window size. This is essentially a 1D
sliding window.

Args:
    iterable (Iterable): an iterable sequence

    size (int): sliding window size (default = 2)

    step (int): sliding step size (default = 1)

    wrap (bool): wraparound (default = False)

Returns:
    iter: returns windows in a sequence

Example:
    >>> iterable = [1, 2, 3, 4, 5, 6]
    >>> size, step, wrap = 3, 1, True
    >>> window_iter = iter_window(iterable, size, step, wrap)
    >>> window_list = list(window_iter)
    >>> print('window_list = %r' % (window_list,))
    window_list = [(1, 2, 3), (2, 3, 4), (3, 4, 5), (4, 5, 6), (5, 6, 1), (6, 1, 2)]

Example:
    >>> iterable = [1, 2, 3, 4, 5, 6]
    >>> size, step, wrap = 3, 2, True
    >>> window_iter = iter_window(iterable, size, step, wrap)
    >>> window_list = list(window_iter)
    >>> print('window_list = %r' % (window_list,))
    window_list = [(1, 2, 3), (3, 4, 5), (5, 6, 1)]

Example:
    >>> iterable = [1, 2, 3, 4, 5, 6]
    >>> size, step, wrap = 3, 2, False
    >>> window_iter = iter_window(iterable, size, step, wrap)
    >>> window_list = list(window_iter)
    >>> print('window_list = %r' % (window_list,))
    window_list = [(1, 2, 3), (3, 4, 5)]

Example:
    >>> iterable = []
    >>> size, step, wrap = 3, 2, False
    >>> window_iter = iter_window(iterable, size, step, wrap)
    >>> window_list = list(window_iter)
    >>> print('window_list = %r' % (window_list,))
    window_list = []


iter_ = iter(iterable)<EOL>try:<EOL><INDENT>first = next(iter_)<EOL><DEDENT>except StopIteration:<EOL><INDENT>return True<EOL><DEDENT>return all(eq(first, item) for item in iter_)<EOL>

Determine if all items in a sequence are the same

Args:
    iterable (Iterable): items to determine if they are all the same

    eq (Callable, optional): function to determine equality
        (default: operator.eq)

Example:
    >>> allsame([1, 1, 1, 1])
    True
    >>> allsame([])
    True
    >>> allsame([0, 1])
    False
    >>> iterable = iter([0, 1, 1, 1])
    >>> next(iterable)
    >>> allsame(iterable)
    True
    >>> allsame(range(10))
    False
    >>> allsame(range(10), lambda a, b: True)
    True


def argsort(indexable, key=None, reverse=False):

<EOL>if isinstance(indexable, collections_abc.Mapping):<EOL><INDENT>vk_iter = ((v, k) for k, v in indexable.items())<EOL><DEDENT>else:<EOL><INDENT>vk_iter = ((v, k) for k, v in enumerate(indexable))<EOL><DEDENT>if key is None:<EOL><INDENT>indices = [k for v, k in sorted(vk_iter, reverse=reverse)]<EOL><DEDENT>else:<EOL><INDENT>indices = [k for v, k in sorted(vk_iter, key=lambda vk: key(vk[<NUM_LIT:0>]),<EOL>reverse=reverse)]<EOL><DEDENT>return indices<EOL>

Returns the indices that would sort a indexable object.

This is similar to `numpy.argsort`, but it is written in pure python and
works on both lists and dictionaries.

Args:
    indexable (Iterable or Mapping): indexable to sort by

    key (Callable, optional): customizes the ordering of the indexable

    reverse (bool, optional): if True returns in descending order

Returns:
    list: indices: list of indices such that sorts the indexable

Example:
    >>> import ubelt as ub
    >>> # argsort works on dicts by returning keys
    >>> dict_ = {'a': 3, 'b': 2, 'c': 100}
    >>> indices = ub.argsort(dict_)
    >>> assert list(ub.take(dict_, indices)) == sorted(dict_.values())
    >>> # argsort works on lists by returning indices
    >>> indexable = [100, 2, 432, 10]
    >>> indices = ub.argsort(indexable)
    >>> assert list(ub.take(indexable, indices)) == sorted(indexable)
    >>> # Can use iterators, but be careful. It exhausts them.
    >>> indexable = reversed(range(100))
    >>> indices = ub.argsort(indexable)
    >>> assert indices[0] == 99
    >>> # Can use key just like sorted
    >>> indexable = [[0, 1, 2], [3, 4], [5]]
    >>> indices = ub.argsort(indexable, key=len)
    >>> assert indices == [2, 1, 0]
    >>> # Can use reverse just like sorted
    >>> indexable = [0, 2, 1]
    >>> indices = ub.argsort(indexable, reverse=True)
    >>> assert indices == [1, 2, 0]


if key is None and isinstance(indexable, collections_abc.Mapping):<EOL><INDENT>return max(indexable.items(), key=operator.itemgetter(<NUM_LIT:1>))[<NUM_LIT:0>]<EOL><DEDENT>elif hasattr(indexable, '<STR_LIT:index>'):<EOL><INDENT>if key is None:<EOL><INDENT>return indexable.index(max(indexable))<EOL><DEDENT>else:<EOL><INDENT>return indexable.index(max(indexable, key=key))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return argsort(indexable, key=key)[-<NUM_LIT:1>]<EOL><DEDENT>

Returns index / key of the item with the largest value.

This is similar to `numpy.argmax`, but it is written in pure python and
works on both lists and dictionaries.

Args:
    indexable (Iterable or Mapping): indexable to sort by

    key (Callable, optional): customizes the ordering of the indexable

CommandLine:
    python -m ubelt.util_list argmax

Example:
    >>> assert argmax({'a': 3, 'b': 2, 'c': 100}) == 'c'
    >>> assert argmax(['a', 'c', 'b', 'z', 'f']) == 3
    >>> assert argmax([[0, 1], [2, 3, 4], [5]], key=len) == 1
    >>> assert argmax({'a': 3, 'b': 2, 3: 100, 4: 4}) == 3
    >>> assert argmax(iter(['a', 'c', 'b', 'z', 'f'])) == 3


if key is None and isinstance(indexable, collections_abc.Mapping):<EOL><INDENT>return min(indexable.items(), key=operator.itemgetter(<NUM_LIT:1>))[<NUM_LIT:0>]<EOL><DEDENT>elif hasattr(indexable, '<STR_LIT:index>'):<EOL><INDENT>if key is None:<EOL><INDENT>return indexable.index(min(indexable))<EOL><DEDENT>else:<EOL><INDENT>return indexable.index(min(indexable, key=key))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return argsort(indexable, key=key)[<NUM_LIT:0>]<EOL><DEDENT>

Returns index / key of the item with the smallest value.

This is similar to `numpy.argmin`, but it is written in pure python and
works on both lists and dictionaries.

Args:
    indexable (Iterable or Mapping): indexable to sort by

    key (Callable, optional): customizes the ordering of the indexable

Example:
    >>> assert argmin({'a': 3, 'b': 2, 'c': 100}) == 'b'
    >>> assert argmin(['a', 'c', 'b', 'z', 'f']) == 0
    >>> assert argmin([[0, 1], [2, 3, 4], [5]], key=len) == 2
    >>> assert argmin({'a': 3, 'b': 2, 3: 100, 4: 4}) == 'b'
    >>> assert argmin(iter(['a', 'c', 'A', 'z', 'f'])) == 2


return next(iter(iterable))<EOL>

Look at the first item of an iterable. If the input is an iterator, then
the next element is exhausted (i.e. a pop operation).

Args:
    iterable (List[T]): an iterable

Returns:
    T: item: the first item of ordered sequence, a popped item from an
             iterator, or an arbitrary item from an unordered collection.

Example:
    >>> import ubelt as ub
    >>> data = [0, 1, 2]
    >>> ub.peek(data)
    0
    >>> iterator = iter(data)
    >>> print(ub.peek(iterator))
    0
    >>> print(ub.peek(iterator))
    1
    >>> print(ub.peek(iterator))
    2
    >>> ub.peek(range(3))
    0


def _symlink(path, link, overwrite=<NUM_LIT:0>, verbose=<NUM_LIT:0>):

if exists(link) and not os.path.islink(link):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>is_junc = _win32_is_junction(link)<EOL>if os.path.isdir(link):<EOL><INDENT>if is_junc:<EOL><INDENT>pointed = _win32_read_junction(link)<EOL>if path == pointed:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>return link<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>if not exists(pointed):<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>raise IOError('<STR_LIT>')<EOL><DEDENT><DEDENT>elif os.path.isfile(link):<EOL><INDENT>if _win32_is_hardlinked(link, path):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>return link<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>if _win32_can_symlink():<EOL><INDENT>raise IOError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>if overwrite:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>util_io.delete(link, verbose > <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>if exists(link):<EOL><INDENT>raise IOError('<STR_LIT>')<EOL><DEDENT><DEDENT><DEDENT>_win32_symlink2(path, link, verbose=verbose)<EOL>


def _win32_symlink2(path, link, allow_fallback=True, verbose=<NUM_LIT:0>):

if _win32_can_symlink():<EOL><INDENT>return _win32_symlink(path, link, verbose)<EOL><DEDENT>else:<EOL><INDENT>return _win32_junction(path, link, verbose)<EOL><DEDENT>

Perform a real symbolic link if possible. However, on most versions of
windows you need special privledges to create a real symlink. Therefore, we
try to create a symlink, but if that fails we fallback to using a junction.

AFAIK, the main difference between symlinks and junctions are that symlinks
can reference relative or absolute paths, where as junctions always
reference absolute paths. Not 100% on this though. Windows is weird.

Note that junctions will not register as links via `islink`, but I
believe real symlinks will.


def _win32_symlink(path, link, verbose=<NUM_LIT:0>):

from ubelt import util_cmd<EOL>if os.path.isdir(path):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>command = '<STR_LIT>'.format(link, path)<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>command = '<STR_LIT>'.format(link, path)<EOL><DEDENT>if command is not None:<EOL><INDENT>info = util_cmd.cmd(command, shell=True)<EOL>if info['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>from ubelt import util_format<EOL>permission_msg = '<STR_LIT>'<EOL>if permission_msg not in info['<STR_LIT>']:<EOL><INDENT>print('<STR_LIT>')<EOL>print(info['<STR_LIT>'])<EOL>print(util_format.repr2(info, nl=<NUM_LIT:1>))<EOL><DEDENT>raise OSError(str(info))<EOL><DEDENT><DEDENT>return link<EOL>

Creates real symlink. This will only work in versions greater than Windows
Vista. Creating real symlinks requires admin permissions or at least
specially enabled symlink permissions. On Windows 10 enabling developer
mode should give you these permissions.


def _win32_junction(path, link, verbose=<NUM_LIT:0>):

<EOL>path = os.path.abspath(path)<EOL>link = os.path.abspath(link)<EOL>from ubelt import util_cmd<EOL>if os.path.isdir(path):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>command = '<STR_LIT>'.format(link, path)<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>try:<EOL><INDENT>jwfs.link(path, link)  <EOL><DEDENT>except Exception:<EOL><INDENT>print('<STR_LIT>'.format(link, path))<EOL>raise<EOL><DEDENT>command = None<EOL><DEDENT>if command is not None:<EOL><INDENT>info = util_cmd.cmd(command, shell=True)<EOL>if info['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>from ubelt import util_format<EOL>print('<STR_LIT>')<EOL>print(info['<STR_LIT>'])<EOL>print(util_format.repr2(info, nl=<NUM_LIT:1>))<EOL>raise OSError(str(info))<EOL><DEDENT><DEDENT>return link<EOL>

On older (pre 10) versions of windows we need admin privledges to make
symlinks, however junctions seem to work.

For paths we do a junction (softlink) and for files we use a hard link

CommandLine:
    python -m ubelt._win32_links _win32_junction

Example:
    >>> # xdoc: +REQUIRES(WIN32)
    >>> import ubelt as ub
    >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')
    >>> ub.delete(root)
    >>> ub.ensuredir(root)
    >>> fpath = join(root, 'fpath.txt')
    >>> dpath = join(root, 'dpath')
    >>> fjunc = join(root, 'fjunc.txt')
    >>> djunc = join(root, 'djunc')
    >>> ub.touch(fpath)
    >>> ub.ensuredir(dpath)
    >>> ub.ensuredir(join(root, 'djunc_fake'))
    >>> ub.ensuredir(join(root, 'djunc_fake with space'))
    >>> ub.touch(join(root, 'djunc_fake with space file'))
    >>> _win32_junction(fpath, fjunc)
    >>> _win32_junction(dpath, djunc)
    >>> # thank god colons are not allowed
    >>> djunc2 = join(root, 'djunc2 [with pathological attrs]')
    >>> _win32_junction(dpath, djunc2)
    >>> _win32_is_junction(djunc)
    >>> ub.writeto(join(djunc, 'afile.txt'), 'foo')
    >>> assert ub.readfrom(join(dpath, 'afile.txt')) == 'foo'
    >>> ub.writeto(fjunc, 'foo')


if not exists(path):<EOL><INDENT>if os.path.isdir(path):<EOL><INDENT>if not os.path.islink(path):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL><DEDENT>return jwfs.is_reparse_point(path) and not os.path.islink(path)<EOL>

Determines if a path is a win32 junction

CommandLine:
    python -m ubelt._win32_links _win32_is_junction

Example:
    >>> # xdoc: +REQUIRES(WIN32)
    >>> import ubelt as ub
    >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')
    >>> ub.delete(root)
    >>> ub.ensuredir(root)
    >>> dpath = join(root, 'dpath')
    >>> djunc = join(root, 'djunc')
    >>> ub.ensuredir(dpath)
    >>> _win32_junction(dpath, djunc)
    >>> assert _win32_is_junction(djunc) is True
    >>> assert _win32_is_junction(dpath) is False
    >>> assert _win32_is_junction('notafile') is False


if not jwfs.is_reparse_point(path):<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>handle = jwfs.api.CreateFile(<EOL>path, <NUM_LIT:0>, <NUM_LIT:0>, None, jwfs.api.OPEN_EXISTING,<EOL>jwfs.api.FILE_FLAG_OPEN_REPARSE_POINT |<EOL>jwfs.api.FILE_FLAG_BACKUP_SEMANTICS,<EOL>None)<EOL>if handle == jwfs.api.INVALID_HANDLE_VALUE:<EOL><INDENT>raise WindowsError()<EOL><DEDENT>res = jwfs.reparse.DeviceIoControl(<EOL>handle, jwfs.api.FSCTL_GET_REPARSE_POINT, None, <NUM_LIT>)<EOL>bytes = jwfs.create_string_buffer(res)<EOL>p_rdb = jwfs.cast(bytes, jwfs.POINTER(jwfs.api.REPARSE_DATA_BUFFER))<EOL>rdb = p_rdb.contents<EOL>if rdb.tag not in [<NUM_LIT>, jwfs.api.IO_REPARSE_TAG_SYMLINK]:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>" % rdb.tag)<EOL><DEDENT>jwfs.handle_nonzero_success(jwfs.api.CloseHandle(handle))<EOL>subname = rdb.get_substitute_name()<EOL>if subname.startswith('<STR_LIT>'):<EOL><INDENT>subname = subname[<NUM_LIT:2>:]<EOL><DEDENT>return subname<EOL>

Returns the location that the junction points, raises ValueError if path is
not a junction.

CommandLine:
    python -m ubelt._win32_links _win32_read_junction

Example:
    >>> # xdoc: +REQUIRES(WIN32)
    >>> import ubelt as ub
    >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_junction')
    >>> ub.delete(root)
    >>> ub.ensuredir(root)
    >>> dpath = join(root, 'dpath')
    >>> djunc = join(root, 'djunc')
    >>> ub.ensuredir(dpath)
    >>> _win32_junction(dpath, djunc)
    >>> path = djunc
    >>> pointed = _win32_read_junction(path)
    >>> print('pointed = {!r}'.format(pointed))


<EOL>def _rmjunctions(root):<EOL><INDENT>subdirs = []<EOL>for name in os.listdir(root):<EOL><INDENT>current = join(root, name)<EOL>if os.path.isdir(current):<EOL><INDENT>if _win32_is_junction(current):<EOL><INDENT>os.rmdir(current)<EOL><DEDENT>elif not os.path.islink(current):<EOL><INDENT>subdirs.append(current)<EOL><DEDENT><DEDENT><DEDENT>for subdir in subdirs:<EOL><INDENT>_rmjunctions(subdir)<EOL><DEDENT><DEDENT>if _win32_is_junction(path):<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.rmdir(path)<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>_rmjunctions(path)<EOL>import shutil<EOL>shutil.rmtree(path)<EOL><DEDENT>

rmtree for win32 that treats junctions like directory symlinks.
The junction removal portion may not be safe on race conditions.

There is a known issue that prevents shutil.rmtree from
deleting directories with junctions.
https://bugs.python.org/issue31226


<EOL>def get_read_handle(fpath):<EOL><INDENT>if os.path.isdir(fpath):<EOL><INDENT>dwFlagsAndAttributes = jwfs.api.FILE_FLAG_BACKUP_SEMANTICS<EOL><DEDENT>else:<EOL><INDENT>dwFlagsAndAttributes = <NUM_LIT:0><EOL><DEDENT>hFile = jwfs.api.CreateFile(fpath, jwfs.api.GENERIC_READ,<EOL>jwfs.api.FILE_SHARE_READ, None,<EOL>jwfs.api.OPEN_EXISTING,<EOL>dwFlagsAndAttributes, None)<EOL>return hFile<EOL><DEDENT>def get_unique_id(hFile):<EOL><INDENT>info = jwfs.api.BY_HANDLE_FILE_INFORMATION()<EOL>res = jwfs.api.GetFileInformationByHandle(hFile, info)<EOL>jwfs.handle_nonzero_success(res)<EOL>unique_id = (info.volume_serial_number, info.file_index_high,<EOL>info.file_index_low)<EOL>return unique_id<EOL><DEDENT>hFile1 = get_read_handle(fpath1)<EOL>hFile2 = get_read_handle(fpath2)<EOL>try:<EOL><INDENT>are_equal = (get_unique_id(hFile1) == get_unique_id(hFile2))<EOL><DEDENT>except Exception:<EOL><INDENT>raise<EOL><DEDENT>finally:<EOL><INDENT>jwfs.api.CloseHandle(hFile1)<EOL>jwfs.api.CloseHandle(hFile2)<EOL><DEDENT>return are_equal<EOL>

Test if two hard links point to the same location

CommandLine:
    python -m ubelt._win32_links _win32_is_hardlinked

Example:
    >>> # xdoc: +REQUIRES(WIN32)
    >>> import ubelt as ub
    >>> root = ub.ensure_app_cache_dir('ubelt', 'win32_hardlink')
    >>> ub.delete(root)
    >>> ub.ensuredir(root)
    >>> fpath1 = join(root, 'fpath1')
    >>> fpath2 = join(root, 'fpath2')
    >>> ub.touch(fpath1)
    >>> ub.touch(fpath2)
    >>> fjunc1 = _win32_junction(fpath1, join(root, 'fjunc1'))
    >>> fjunc2 = _win32_junction(fpath2, join(root, 'fjunc2'))
    >>> assert _win32_is_hardlinked(fjunc1, fpath1)
    >>> assert _win32_is_hardlinked(fjunc2, fpath2)
    >>> assert not _win32_is_hardlinked(fjunc2, fpath1)
    >>> assert not _win32_is_hardlinked(fjunc1, fpath2)


from ubelt import util_cmd<EOL>import re<EOL>wrapper = '<STR_LIT>'  <EOL>command = '<STR_LIT>'.format(path, star)<EOL>wrapped = wrapper.format(command)<EOL>info = util_cmd.cmd(wrapped, shell=True)<EOL>if info['<STR_LIT>'] != <NUM_LIT:0>:<EOL><INDENT>from ubelt import util_format<EOL>print('<STR_LIT>')<EOL>print(info['<STR_LIT>'])<EOL>print(util_format.repr2(info, nl=<NUM_LIT:1>))<EOL>raise OSError(str(info))<EOL><DEDENT>lines = info['<STR_LIT>'].split('<STR_LIT:\n>')[<NUM_LIT:5>:-<NUM_LIT:3>]<EOL>splitter = re.compile('<STR_LIT>')<EOL>for line in lines:<EOL><INDENT>parts = splitter.split(line)<EOL>date, sep, time, sep, ampm, sep, type_or_size, sep = parts[:<NUM_LIT:8>]<EOL>name = '<STR_LIT>'.join(parts[<NUM_LIT:8>:])<EOL>if name == '<STR_LIT:.>' or name == '<STR_LIT:..>':<EOL><INDENT>continue<EOL><DEDENT>if type_or_size in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>pos = name.find('<STR_LIT::>')<EOL>bpos = name[:pos].rfind('<STR_LIT:[>')<EOL>name = name[:bpos - <NUM_LIT:1>]<EOL>pointed = name[bpos + <NUM_LIT:1>:-<NUM_LIT:1>]<EOL>yield type_or_size, name, pointed<EOL><DEDENT>else:<EOL><INDENT>yield type_or_size, name, None<EOL><DEDENT><DEDENT>

Using the windows cmd shell to get information about a directory


def argval(key, default=util_const.NoParam, argv=None):

if argv is None:  <EOL><INDENT>argv = sys.argv<EOL><DEDENT>keys = [key] if isinstance(key, six.string_types) else key<EOL>n_max = len(argv) - <NUM_LIT:1><EOL>for argx, item in enumerate(argv):<EOL><INDENT>for key_ in keys:<EOL><INDENT>if item == key_:<EOL><INDENT>if argx < n_max:<EOL><INDENT>value = argv[argx + <NUM_LIT:1>]<EOL>return value<EOL><DEDENT><DEDENT>elif item.startswith(key_ + '<STR_LIT:=>'):<EOL><INDENT>value = '<STR_LIT:=>'.join(item.split('<STR_LIT:=>')[<NUM_LIT:1>:])<EOL>return value<EOL><DEDENT><DEDENT><DEDENT>value = default<EOL>return value<EOL>

Get the value of a keyword argument specified on the command line.

Values can be specified as `<key> <value>` or `<key>=<value>`

Args:
    key (str or tuple): string or tuple of strings. Each key should be
        prefixed with two hyphens (i.e. `--`)
    default (Optional[object]): value to return if not specified
    argv (Optional[list]): overrides `sys.argv` if specified

Returns:
    str: value : the value specified after the key. It they key is
        specified multiple times, then the first value is returned.

TODO:
    - [ ] Can we handle the case where the value is a list of long paths?
    - [ ] Should we default the first or last specified instance of the flag.

Example:
    >>> import ubelt as ub
    >>> argv = ['--ans', '42', '--quest=the grail', '--ans=6', '--bad']
    >>> assert ub.argval('--spam', argv=argv) == ub.NoParam
    >>> assert ub.argval('--quest', argv=argv) == 'the grail'
    >>> assert ub.argval('--ans', argv=argv) == '42'
    >>> assert ub.argval('--bad', argv=argv) == ub.NoParam
    >>> assert ub.argval(('--bad', '--bar'), argv=argv) == ub.NoParam

Example:
    >>> # Test fix for GH Issue #41
    >>> import ubelt as ub
    >>> argv = ['--path=/path/with/k=3']
    >>> ub.argval('--path', argv=argv) == '/path/with/k=3'


if argv is None:  <EOL><INDENT>argv = sys.argv<EOL><DEDENT>keys = [key] if isinstance(key, six.string_types) else key<EOL>flag = any(k in argv for k in keys)<EOL>return flag<EOL>

Determines if a key is specified on the command line

Args:
    key (str or tuple): string or tuple of strings. Each key should be
        prefixed with two hyphens (i.e. `--`)
    argv (Optional[list]): overrides `sys.argv` if specified

Returns:
    bool: flag : True if the key (or any of the keys) was specified

Example:
    >>> import ubelt as ub
    >>> argv = ['--spam', '--eggs', 'foo']
    >>> assert ub.argflag('--eggs', argv=argv) is True
    >>> assert ub.argflag('--ans', argv=argv) is False
    >>> assert ub.argflag('foo', argv=argv) is True
    >>> assert ub.argflag(('bar', '--spam'), argv=argv) is True


def writeto(fpath, to_write, aslines=False, verbose=None):

if verbose:<EOL><INDENT>print('<STR_LIT>' % (fpath,))<EOL><DEDENT>with open(fpath, '<STR_LIT:wb>') as file:<EOL><INDENT>if aslines:<EOL><INDENT>to_write = map(_ensure_bytes , to_write)<EOL>file.writelines(to_write)<EOL><DEDENT>else:<EOL><INDENT>bytes = _ensure_bytes(to_write)<EOL>file.write(bytes)<EOL><DEDENT><DEDENT>

r"""
    Writes (utf8) text to a file.

    Args:
        fpath (PathLike): file path
        to_write (str): text to write (must be unicode text)
        aslines (bool): if True to_write is assumed to be a list of lines
        verbose (bool): verbosity flag

    CommandLine:
        python -m ubelt.util_io writeto --verbose

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt')
        >>> fpath = dpath + '/' + 'testwrite.txt'
        >>> if exists(fpath):
        >>>     os.remove(fpath)
        >>> to_write = 'utf-8 symbols Δ, Й, ק, م, ๗, あ, 叶, 葉, and 말.'
        >>> writeto(fpath, to_write)
        >>> read_ = ub.readfrom(fpath)
        >>> print('read_    = ' + read_)
        >>> print('to_write = ' + to_write)
        >>> assert read_ == to_write

    Example:
        >>> import ubelt as ub
        >>> dpath = ub.ensure_app_cache_dir('ubelt')
        >>> fpath = dpath + '/' + 'testwrite2.txt'
        >>> if exists(fpath):
        >>>     os.remove(fpath)
        >>> to_write = ['a\n', 'b\n', 'c\n', 'd\n']
        >>> writeto(fpath, to_write, aslines=True)
        >>> read_ = ub.readfrom(fpath, aslines=True)
        >>> print('read_    = {}'.format(read_))
        >>> print('to_write = {}'.format(to_write))
        >>> assert read_ == to_write


def readfrom(fpath, aslines=False, errors='<STR_LIT:replace>', verbose=None):

if verbose:<EOL><INDENT>print('<STR_LIT>' % (fpath,))<EOL><DEDENT>if not exists(fpath):<EOL><INDENT>raise IOError('<STR_LIT>' % (fpath,))<EOL><DEDENT>with open(fpath, '<STR_LIT:rb>') as file:<EOL><INDENT>if aslines:<EOL><INDENT>text = [line.decode('<STR_LIT:utf8>', errors=errors)<EOL>for line in file.readlines()]<EOL>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>text = [<EOL>line[:-<NUM_LIT:2>] + '<STR_LIT:\n>' if line.endswith('<STR_LIT:\r\n>') else line<EOL>for line in text<EOL>]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>text = file.read().decode('<STR_LIT:utf8>', errors=errors)<EOL><DEDENT><DEDENT>return text<EOL>

Reads (utf8) text from a file.

Args:
    fpath (PathLike): file path
    aslines (bool): if True returns list of lines
    verbose (bool): verbosity flag

Returns:
    str: text from fpath (this is unicode)


def touch(fpath, mode=<NUM_LIT>, dir_fd=None, verbose=<NUM_LIT:0>, **kwargs):

if verbose:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>if six.PY2:  <EOL><INDENT>with open(fpath, '<STR_LIT:a>'):<EOL><INDENT>os.utime(fpath, None)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>flags = os.O_CREAT | os.O_APPEND<EOL>with os.fdopen(os.open(fpath, flags=flags, mode=mode, dir_fd=dir_fd)) as f:<EOL><INDENT>os.utime(f.fileno() if os.utime in os.supports_fd else fpath,<EOL>dir_fd=None if os.supports_fd else dir_fd, **kwargs)<EOL><DEDENT><DEDENT>return fpath<EOL>

change file timestamps

Works like the touch unix utility

Args:
    fpath (PathLike): name of the file
    mode (int): file permissions (python3 and unix only)
    dir_fd (file): optional directory file descriptor. If specified, fpath
        is interpreted as relative to this descriptor (python 3 only).
    verbose (int): verbosity
    **kwargs : extra args passed to `os.utime` (python 3 only).

Returns:
    PathLike: path to the file

References:
    https://stackoverflow.com/questions/1158076/implement-touch-using-python

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_cache_dir('ubelt')
    >>> fpath = join(dpath, 'touch_file')
    >>> assert not exists(fpath)
    >>> ub.touch(fpath)
    >>> assert exists(fpath)
    >>> os.unlink(fpath)


if not os.path.exists(path):<EOL><INDENT>if os.path.islink(path):<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.unlink(path)<EOL><DEDENT>elif os.path.isdir(path):  <EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.rmdir(path)<EOL><DEDENT>elif os.path.isfile(path):  <EOL><INDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.unlink(path)<EOL><DEDENT>else:<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if os.path.islink(path):<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.unlink(path)<EOL><DEDENT>elif os.path.isfile(path):<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>os.unlink(path)<EOL><DEDENT>elif os.path.isdir(path):<EOL><INDENT>if verbose:  <EOL><INDENT>print('<STR_LIT>'.format(path))<EOL><DEDENT>if sys.platform.startswith('<STR_LIT:win32>'):  <EOL><INDENT>from ubelt import _win32_links<EOL>_win32_links._win32_rmtree(path, verbose=verbose)<EOL><DEDENT>else:<EOL><INDENT>import shutil<EOL>shutil.rmtree(path)<EOL><DEDENT><DEDENT><DEDENT>

Removes a file or recursively removes a directory.
If a path does not exist, then this is does nothing.

Args:
    path (PathLike): file or directory to remove
    verbose (bool): if True prints what is being done

SeeAlso:
    send2trash - A cross-platform Python package for sending files
        to the trash instead of irreversibly deleting them.
        https://github.com/hsoft/send2trash

Doctest:
    >>> import ubelt as ub
    >>> base = ub.ensure_app_cache_dir('ubelt', 'delete_test')
    >>> dpath1 = ub.ensuredir(join(base, 'dir'))
    >>> ub.ensuredir(join(base, 'dir', 'subdir'))
    >>> ub.touch(join(base, 'dir', 'to_remove1.txt'))
    >>> fpath1 = join(base, 'dir', 'subdir', 'to_remove3.txt')
    >>> fpath2 = join(base, 'dir', 'subdir', 'to_remove2.txt')
    >>> ub.touch(fpath1)
    >>> ub.touch(fpath2)
    >>> assert all(map(exists, (dpath1, fpath1, fpath2)))
    >>> ub.delete(fpath1)
    >>> assert all(map(exists, (dpath1, fpath2)))
    >>> assert not exists(fpath1)
    >>> ub.delete(dpath1)
    >>> assert not any(map(exists, (dpath1, fpath1, fpath2)))

Doctest:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_cache_dir('ubelt', 'delete_test2')
    >>> dpath1 = ub.ensuredir(join(dpath, 'dir'))
    >>> fpath1 = ub.touch(join(dpath1, 'to_remove.txt'))
    >>> assert exists(fpath1)
    >>> ub.delete(dpath)
    >>> assert not exists(fpath1)


try:<EOL><INDENT>len(items1)<EOL><DEDENT>except TypeError:<EOL><INDENT>items1 = list(items1)<EOL><DEDENT>try:<EOL><INDENT>len(items2)<EOL><DEDENT>except TypeError:<EOL><INDENT>items2 = list(items2)<EOL><DEDENT>if len(items1) == <NUM_LIT:0> and len(items2) == <NUM_LIT:1>:<EOL><INDENT>items2 = []<EOL><DEDENT>if len(items2) == <NUM_LIT:1> and len(items1) > <NUM_LIT:1>:<EOL><INDENT>items2 = items2 * len(items1)<EOL><DEDENT>if len(items1) != len(items2):<EOL><INDENT>raise ValueError('<STR_LIT>' % (<EOL>len(items1), len(items2)))<EOL><DEDENT>return cls(zip(items1, items2))<EOL>

Zips elementwise pairs between items1 and items2 into a dictionary. Values
from items2 can be broadcast onto items1.

Args:
    items1 (Iterable): full sequence
    items2 (Iterable): can either be a sequence of one item or a sequence
        of equal length to `items1`
    cls (Type[dict]): dictionary type to use. Defaults to dict, but could
        be ordered dict instead.

Returns:
    dict: similar to dict(zip(items1, items2))

Example:
    >>> assert dzip([1, 2, 3], [4]) == {1: 4, 2: 4, 3: 4}
    >>> assert dzip([1, 2, 3], [4, 4, 4]) == {1: 4, 2: 4, 3: 4}
    >>> assert dzip([], [4]) == {}


if callable(groupids):<EOL><INDENT>keyfunc = groupids<EOL>pair_list = ((keyfunc(item), item) for item in items)<EOL><DEDENT>else:<EOL><INDENT>pair_list = zip(groupids, items)<EOL><DEDENT>groupid_to_items = defaultdict(list)<EOL>for key, item in pair_list:<EOL><INDENT>groupid_to_items[key].append(item)<EOL><DEDENT>return groupid_to_items<EOL>

r"""
    Groups a list of items by group id.

    Args:
        items (Iterable): a list of items to group
        groupids (Iterable or Callable): a corresponding list of item groupids
            or a function mapping an item to a groupid.

    Returns:
        dict: groupid_to_items: maps a groupid to a list of items

    CommandLine:
        python -m ubelt.util_dict group_items

    Example:
        >>> import ubelt as ub
        >>> items    = ['ham',     'jam',   'spam',     'eggs',    'cheese', 'banana']
        >>> groupids = ['protein', 'fruit', 'protein',  'protein', 'dairy',  'fruit']
        >>> groupid_to_items = ub.group_items(items, groupids)
        >>> print(ub.repr2(groupid_to_items, nl=0))
        {'dairy': ['cheese'], 'fruit': ['jam', 'banana'], 'protein': ['ham', 'spam', 'eggs']}


def dict_hist(item_list, weight_list=None, ordered=False, labels=None):

if labels is None:<EOL><INDENT>hist_ = defaultdict(lambda: <NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>hist_ = {k: <NUM_LIT:0> for k in labels}<EOL><DEDENT>if weight_list is None:<EOL><INDENT>weight_list = it.repeat(<NUM_LIT:1>)<EOL><DEDENT>for item, weight in zip(item_list, weight_list):<EOL><INDENT>hist_[item] += weight<EOL><DEDENT>if ordered:<EOL><INDENT>getval = op.itemgetter(<NUM_LIT:1>)<EOL>hist = OrderedDict([<EOL>(key, value)<EOL>for (key, value) in sorted(hist_.items(), key=getval)<EOL>])<EOL><DEDENT>else:<EOL><INDENT>hist = dict(hist_)<EOL><DEDENT>return hist<EOL>

Builds a histogram of items, counting the number of time each item appears
in the input.

Args:
    item_list (Iterable): hashable items (usually containing duplicates)
    weight_list (Iterable): corresponding weights for each item
    ordered (bool): if True the result is ordered by frequency
    labels (Iterable, optional): expected labels (default None)
        Allows this function to pre-initialize the histogram.
        If specified the frequency of each label is initialized to
        zero and item_list can only contain items specified in labels.

Returns:
    dict : dictionary where the keys are items in item_list, and the values
      are the number of times the item appears in item_list.

CommandLine:
    python -m ubelt.util_dict dict_hist

Example:
    >>> import ubelt as ub
    >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900]
    >>> hist = ub.dict_hist(item_list)
    >>> print(ub.repr2(hist, nl=0))
    {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2}

Example:
    >>> import ubelt as ub
    >>> item_list = [1, 2, 39, 900, 1232, 900, 1232, 2, 2, 2, 900]
    >>> hist1 = ub.dict_hist(item_list)
    >>> hist2 = ub.dict_hist(item_list, ordered=True)
    >>> try:
    >>>     hist3 = ub.dict_hist(item_list, labels=[])
    >>> except KeyError:
    >>>     pass
    >>> else:
    >>>     raise AssertionError('expected key error')
    >>> #result = ub.repr2(hist_)
    >>> weight_list = [1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]
    >>> hist4 = ub.dict_hist(item_list, weight_list=weight_list)
    >>> print(ub.repr2(hist1, nl=0))
    {1: 1, 2: 4, 39: 1, 900: 3, 1232: 2}
    >>> print(ub.repr2(hist4, nl=0))
    {1: 1, 2: 4, 39: 1, 900: 1, 1232: 0}


def find_duplicates(items, k=<NUM_LIT:2>, key=None):

<EOL>duplicates = defaultdict(list)<EOL>if key is None:<EOL><INDENT>for count, item in enumerate(items):<EOL><INDENT>duplicates[item].append(count)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for count, item in enumerate(items):<EOL><INDENT>duplicates[key(item)].append(count)<EOL><DEDENT><DEDENT>for key in list(duplicates.keys()):<EOL><INDENT>if len(duplicates[key]) < k:<EOL><INDENT>del duplicates[key]<EOL><DEDENT><DEDENT>duplicates = dict(duplicates)<EOL>return duplicates<EOL>

Find all duplicate items in a list.

Search for all items that appear more than `k` times and return a mapping
from each (k)-duplicate item to the positions it appeared in.

Args:
    items (Iterable): hashable items possibly containing duplicates
    k (int): only return items that appear at least `k` times (default=2)
    key (Callable, optional): Returns indices where `key(items[i])`
        maps to a particular value at least k times.

Returns:
    dict: maps each duplicate item to the indices at which it appears

CommandLine:
    python -m ubelt.util_dict find_duplicates

Example:
    >>> import ubelt as ub
    >>> items = [0, 0, 1, 2, 3, 3, 0, 12, 2, 9]
    >>> duplicates = ub.find_duplicates(items)
    >>> print('items = %r' % (items,))
    >>> print('duplicates = %r' % (duplicates,))
    >>> assert duplicates == {0: [0, 1, 6], 2: [3, 8], 3: [4, 5]}
    >>> assert ub.find_duplicates(items, 3) == {0: [0, 1, 6]}

Example:
    >>> import ubelt as ub
    >>> items = [0, 0, 1, 2, 3, 3, 0, 12, 2, 9]
    >>> # note: k can be 0
    >>> duplicates = ub.find_duplicates(items, k=0)
    >>> print(ub.repr2(duplicates, nl=0))
    {0: [0, 1, 6], 1: [2], 2: [3, 8], 3: [4, 5], 9: [9], 12: [7]}

Example:
    >>> import ubelt as ub
    >>> items = [10, 11, 12, 13, 14, 15, 16]
    >>> duplicates = ub.find_duplicates(items, key=lambda x: x // 2)
    >>> print(ub.repr2(duplicates, nl=0))
    {5: [0, 1], 6: [2, 3], 7: [4, 5]}


def dict_subset(dict_, keys, default=util_const.NoParam):

keys = list(keys)<EOL>items = dict_take(dict_, keys, default)<EOL>subdict_ = OrderedDict(list(zip(keys, items)))<EOL>return subdict_<EOL>

Get a subset of a dictionary

Args:
    dict_ (Mapping): superset dictionary
    keys (Iterable): keys to take from `dict_`
    default (object, optional): if specified uses default if keys are missing

Returns:
    OrderedDict: subset dictionary

SeeAlso:
    dict_isect - similar functionality, but will only take existing keys

Example:
    >>> import ubelt as ub
    >>> dict_ = {'K': 3, 'dcvs_clip_max': 0.2, 'p': 0.1}
    >>> keys = ['K', 'dcvs_clip_max']
    >>> subdict_ = ub.dict_subset(dict_, keys)
    >>> print(ub.repr2(subdict_, nl=0))
    {'K': 3, 'dcvs_clip_max': 0.2}


def dict_take(dict_, keys, default=util_const.NoParam):

if default is util_const.NoParam:<EOL><INDENT>for key in keys:<EOL><INDENT>yield dict_[key]<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for key in keys:<EOL><INDENT>yield dict_.get(key, default)<EOL><DEDENT><DEDENT>

r"""
    Generates values from a dictionary

    Args:
        dict_ (Mapping): a dictionary to take from
        keys (Iterable): the keys to take
        default (object, optional): if specified uses default if keys are missing

    CommandLine:
        python -m ubelt.util_dict dict_take_gen

    Example:
        >>> import ubelt as ub
        >>> dict_ = {1: 'a', 2: 'b', 3: 'c'}
        >>> keys = [1, 2, 3, 4, 5]
        >>> result = list(ub.dict_take(dict_, keys, None))
        >>> assert result == ['a', 'b', 'c', None, None]

    Example:
        >>> import ubelt as ub
        >>> dict_ = {1: 'a', 2: 'b', 3: 'c'}
        >>> keys = [1, 2, 3, 4, 5]
        >>> try:
        >>>     print(list(ub.dict_take(dict_, keys)))
        >>>     raise AssertionError('did not get key error')
        >>> except KeyError:
        >>>     print('correctly got key error')


if not args:<EOL><INDENT>return {}<EOL><DEDENT>else:<EOL><INDENT>dictclass = OrderedDict if isinstance(args[<NUM_LIT:0>], OrderedDict) else dict<EOL>return dictclass(it.chain.from_iterable(d.items() for d in args))<EOL><DEDENT>

Combines the disjoint keys in multiple dictionaries. For intersecting keys,
dictionaries towards the end of the sequence are given precedence.

Args:
    *args : a sequence of dictionaries

Returns:
    Dict | OrderedDict :
        OrderedDict if the first argument is an OrderedDict, otherwise dict

SeeAlso:
    collections.ChainMap - a standard python builtin data structure that
        provides a view that treats multiple dicts as a single dict.
        https://docs.python.org/3/library/collections.html#chainmap-objects

Example:
    >>> result = dict_union({'a': 1, 'b': 1}, {'b': 2, 'c': 2})
    >>> assert result == {'a': 1, 'b': 2, 'c': 2}
    >>> dict_union(odict([('a', 1), ('b', 2)]), odict([('c', 3), ('d', 4)]))
    OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4)])
    >>> dict_union()
    {}


if not args:<EOL><INDENT>return {}<EOL><DEDENT>else:<EOL><INDENT>dictclass = OrderedDict if isinstance(args[<NUM_LIT:0>], OrderedDict) else dict<EOL>common_keys = set.intersection(*map(set, args))<EOL>first_dict = args[<NUM_LIT:0>]<EOL>return dictclass((k, first_dict[k]) for k in common_keys)<EOL><DEDENT>

Constructs a dictionary that contains keys common between all inputs.
The returned values will only belong to the first dictionary.

Args:
    *args : a sequence of dictionaries (or sets of keys)

Returns:
    Dict | OrderedDict :
        OrderedDict if the first argument is an OrderedDict, otherwise dict

Notes:
    This function can be used as an alternative to `dict_subset` where any
    key not in the dictionary is ignored. See the following example:

    >>> dict_isect({'a': 1, 'b': 2, 'c': 3}, ['a', 'c', 'd'])
    {'a': 1, 'c': 3}

Example:
    >>> dict_isect({'a': 1, 'b': 1}, {'b': 2, 'c': 2})
    {'b': 1}
    >>> dict_isect(odict([('a', 1), ('b', 2)]), odict([('c', 3)]))
    OrderedDict()
    >>> dict_isect()
    {}


if not hasattr(func, '<STR_LIT>'):<EOL><INDENT>func = func.__getitem__<EOL><DEDENT>keyval_list = [(key, func(val)) for key, val in six.iteritems(dict_)]<EOL>dictclass = OrderedDict if isinstance(dict_, OrderedDict) else dict<EOL>newdict = dictclass(keyval_list)<EOL>return newdict<EOL>

applies a function to each of the keys in a dictionary

Args:
    func (callable): a function or indexable object
    dict_ (dict): a dictionary

Returns:
    newdict: transformed dictionary

CommandLine:
    python -m ubelt.util_dict map_vals

Example:
    >>> import ubelt as ub
    >>> dict_ = {'a': [1, 2, 3], 'b': []}
    >>> func = len
    >>> newdict = ub.map_vals(func, dict_)
    >>> assert newdict ==  {'a': 3, 'b': 0}
    >>> print(newdict)
    >>> # Can also use indexables as `func`
    >>> dict_ = {'a': 0, 'b': 1}
    >>> func = [42, 21]
    >>> newdict = ub.map_vals(func, dict_)
    >>> assert newdict ==  {'a': 42, 'b': 21}
    >>> print(newdict)


if not hasattr(func, '<STR_LIT>'):<EOL><INDENT>func = func.__getitem__<EOL><DEDENT>keyval_list = [(func(key), val) for key, val in six.iteritems(dict_)]<EOL>dictclass = OrderedDict if isinstance(dict_, OrderedDict) else dict<EOL>newdict = dictclass(keyval_list)<EOL>assert len(newdict) == len(dict_), (<EOL>'<STR_LIT>')<EOL>return newdict<EOL>

r"""
    applies a function to each of the keys in a dictionary

    Args:
        func (callable): a function or indexable object
        dict_ (dict): a dictionary

    Returns:
        newdict: transformed dictionary

    CommandLine:
        python -m ubelt.util_dict map_keys

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': [1, 2, 3], 'b': []}
        >>> func = ord
        >>> newdict = ub.map_keys(func, dict_)
        >>> print(newdict)
        >>> assert newdict == {97: [1, 2, 3], 98: []}
        >>> #ut.assert_raises(AssertionError, map_keys, len, dict_)
        >>> dict_ = {0: [1, 2, 3], 1: []}
        >>> func = ['a', 'b']
        >>> newdict = ub.map_keys(func, dict_)
        >>> print(newdict)
        >>> assert newdict == {'a': [1, 2, 3], 'b': []}
        >>> #ut.assert_raises(AssertionError, map_keys, len, dict_)


if unique_vals:<EOL><INDENT>if isinstance(dict_, OrderedDict):<EOL><INDENT>inverted = OrderedDict((val, key) for key, val in dict_.items())<EOL><DEDENT>else:<EOL><INDENT>inverted = {val: key for key, val in dict_.items()}<EOL><DEDENT><DEDENT>else:<EOL><INDENT>inverted = defaultdict(set)<EOL>for key, value in dict_.items():<EOL><INDENT>inverted[value].add(key)<EOL><DEDENT>inverted = dict(inverted)<EOL><DEDENT>return inverted<EOL>

r"""
    Swaps the keys and values in a dictionary.

    Args:
        dict_ (dict): dictionary to invert
        unique_vals (bool): if False, inverted keys are returned in a set.
            The default is True.

    Returns:
        dict: inverted

    Notes:
        The must values be hashable.

        If the original dictionary contains duplicate values, then only one of
        the corresponding keys will be returned and the others will be
        discarded.  This can be prevented by setting `unique_vals=True`,
        causing the inverted keys to be returned in a set.

    CommandLine:
        python -m ubelt.util_dict invert_dict

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': 1, 'b': 2}
        >>> inverted = ub.invert_dict(dict_)
        >>> assert inverted == {1: 'a', 2: 'b'}

    Example:
        >>> import ubelt as ub
        >>> dict_ = ub.odict([(2, 'a'), (1, 'b'), (0, 'c'), (None, 'd')])
        >>> inverted = ub.invert_dict(dict_)
        >>> assert list(inverted.keys())[0] == 'a'

    Example:
        >>> import ubelt as ub
        >>> dict_ = {'a': 1, 'b': 0, 'c': 0, 'd': 0, 'f': 2}
        >>> inverted = ub.invert_dict(dict_, unique_vals=False)
        >>> assert inverted == {0: {'b', 'c', 'd'}, 1: {'a'}, 2: {'f'}}


return self._base(<EOL>(key, (value.to_dict() if isinstance(value, AutoDict) else value))<EOL>for key, value in self.items())<EOL>

Recursively casts a AutoDict into a regular dictionary. All nested
AutoDict values are also converted.

Returns:
    dict: a copy of this dict without autovivification

Example:
    >>> from ubelt.util_dict import AutoDict
    >>> auto = AutoDict()
    >>> auto[1] = 1
    >>> auto['n1'] = AutoDict()
    >>> static = auto.to_dict()
    >>> assert not isinstance(static, AutoDict)
    >>> assert not isinstance(static['n1'], AutoDict)


if LINUX:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>elif DARWIN:  <EOL><INDENT>dpath_  = '<STR_LIT>'<EOL><DEDENT>elif WIN32:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:  <EOL><INDENT>raise '<STR_LIT>'<EOL><DEDENT>dpath = normpath(expanduser(dpath_))<EOL>return dpath<EOL>

Returns path for user-specific data files

Returns:
    PathLike : path to the data dir used by the current operating system


if LINUX:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>elif DARWIN:  <EOL><INDENT>dpath_  = '<STR_LIT>'<EOL><DEDENT>elif WIN32:  <EOL><INDENT>dpath_ = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL><DEDENT>else:  <EOL><INDENT>raise NotImplementedError('<STR_LIT>' % (sys.platform,))<EOL><DEDENT>dpath = normpath(expanduser(dpath_))<EOL>return dpath<EOL>

Returns a directory which should be writable for any application
This should be used for persistent configuration files.

Returns:
    PathLike : path to the cahce dir used by the current operating system


Returns a directory which should be writable for any application
This should be used for temporary deletable data.

Returns:
    PathLike : path to the cache dir used by the current operating system


dpath = join(platform_data_dir(), appname, *args)<EOL>return dpath<EOL>

r"""
    Returns a writable directory for an application.
    This should be used for temporary deletable data.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable data directory for this application

    SeeAlso:
        ensure_app_data_dir


from ubelt import util_path<EOL>dpath = get_app_data_dir(appname, *args)<EOL>util_path.ensuredir(dpath)<EOL>return dpath<EOL>

Calls `get_app_data_dir` but ensures the directory exists.

Args:
    appname (str): the name of the application
    *args: any other subdirectories may be specified

SeeAlso:
    get_app_data_dir

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_data_dir('ubelt')
    >>> assert exists(dpath)


dpath = join(platform_config_dir(), appname, *args)<EOL>return dpath<EOL>

r"""
    Returns a writable directory for an application
    This should be used for persistent configuration files.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable config directory for this application

    SeeAlso:
        ensure_app_config_dir


from ubelt import util_path<EOL>dpath = get_app_config_dir(appname, *args)<EOL>util_path.ensuredir(dpath)<EOL>return dpath<EOL>

Calls `get_app_config_dir` but ensures the directory exists.

Args:
    appname (str): the name of the application
    *args: any other subdirectories may be specified

SeeAlso:
    get_app_config_dir

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_config_dir('ubelt')
    >>> assert exists(dpath)


dpath = join(platform_cache_dir(), appname, *args)<EOL>return dpath<EOL>

r"""
    Returns a writable directory for an application.
    This should be used for temporary deletable data.

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable cache directory for this application

    SeeAlso:
        ensure_app_cache_dir


from ubelt import util_path<EOL>dpath = get_app_cache_dir(appname, *args)<EOL>util_path.ensuredir(dpath)<EOL>return dpath<EOL>

Calls `get_app_cache_dir` but ensures the directory exists.

Args:
    appname (str): the name of the application
    *args: any other subdirectories may be specified

SeeAlso:
    get_app_cache_dir

Example:
    >>> import ubelt as ub
    >>> dpath = ub.ensure_app_cache_dir('ubelt')
    >>> assert exists(dpath)


from ubelt import util_cmd<EOL>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>fpath = normpath(fpath)<EOL>if not exists(fpath):<EOL><INDENT>raise Exception('<STR_LIT>' % fpath)<EOL><DEDENT>if not WIN32:<EOL><INDENT>import pipes<EOL>fpath = pipes.quote(fpath)<EOL><DEDENT>if LINUX:<EOL><INDENT>info = util_cmd.cmd(('<STR_LIT>', fpath), detach=True, verbose=verbose)<EOL><DEDENT>elif DARWIN:<EOL><INDENT>info = util_cmd.cmd(('<STR_LIT>', fpath), detach=True, verbose=verbose)<EOL><DEDENT>elif WIN32:<EOL><INDENT>os.startfile(fpath)<EOL>info = None<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>if info is not None:<EOL><INDENT>if not info['<STR_LIT>']:<EOL><INDENT>raise Exception('<STR_LIT>')<EOL><DEDENT><DEDENT>

Uses default program defined by the system to open a file.
This is done via `os.startfile` on windows, `open` on mac, and `xdg-open`
on linux.

Args:
    fpath (PathLike): a file to open using the program associated with the
        files extension type.
    verbose (int): verbosity

References:
    http://stackoverflow.com/questions/2692873/quote-posix

DisableExample:
    >>> # This test interacts with a GUI frontend, not sure how to test.
    >>> import ubelt as ub
    >>> base = ub.ensure_app_cache_dir('ubelt')
    >>> fpath1 = join(base, 'test_open.txt')
    >>> ub.touch(fpath1)
    >>> proc = ub.startfile(fpath1)


def find_exe(name, multi=False, path=None):

candidates = find_path(name, path=path, exact=True)<EOL>mode = os.X_OK | os.F_OK<EOL>results = (fpath for fpath in candidates<EOL>if os.access(fpath, mode) and not isdir(fpath))<EOL>if not multi:<EOL><INDENT>for fpath in results:<EOL><INDENT>return fpath<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return list(results)<EOL><DEDENT>

Locate a command.

Search your local filesystem for an executable and return the first
matching file with executable permission.

Args:
    name (str): globstr of matching filename

    multi (bool): if True return all matches instead of just the first.
        Defaults to False.

    path (str or Iterable[PathLike]): overrides the system PATH variable.

Returns:
    PathLike or List[PathLike] or None: returns matching executable(s).

SeeAlso:
    shutil.which - which is available in Python 3.3+.

Notes:
    This is essentially the `which` UNIX command

References:
    https://stackoverflow.com/questions/377017/test-if-executable-exists-in-python/377028#377028
    https://docs.python.org/dev/library/shutil.html#shutil.which

Example:
    >>> find_exe('ls')
    >>> find_exe('ping')
    >>> assert find_exe('which') == find_exe(find_exe('which'))
    >>> find_exe('which', multi=True)
    >>> find_exe('ping', multi=True)
    >>> find_exe('cmake', multi=True)
    >>> find_exe('nvcc', multi=True)
    >>> find_exe('noexist', multi=True)

Example:
    >>> assert not find_exe('noexist', multi=False)
    >>> assert find_exe('ping', multi=False)
    >>> assert not find_exe('noexist', multi=True)
    >>> assert find_exe('ping', multi=True)

Benchmark:
    >>> # xdoctest: +IGNORE_WANT
    >>> import ubelt as ub
    >>> import shutil
    >>> for timer in ub.Timerit(100, bestof=10, label='ub.find_exe'):
    >>>     ub.find_exe('which')
    >>> for timer in ub.Timerit(100, bestof=10, label='shutil.which'):
    >>>     shutil.which('which')
    Timed best=58.71 µs, mean=59.64 ± 0.96 µs for ub.find_exe
    Timed best=72.75 µs, mean=73.07 ± 0.22 µs for shutil.which


def find_path(name, path=None, exact=False):

path = os.environ.get('<STR_LIT>', os.defpath) if path is None else path<EOL>dpaths = path.split(os.pathsep) if isinstance(path, six.string_types) else path<EOL>candidates = (join(dpath, name) for dpath in dpaths)<EOL>if exact:<EOL><INDENT>if WIN32:  <EOL><INDENT>pathext = ['<STR_LIT>'] + os.environ.get('<STR_LIT>', '<STR_LIT>').split(os.pathsep)<EOL>candidates = (p + ext for p in candidates for ext in pathext)<EOL><DEDENT>candidates = filter(exists, candidates)<EOL><DEDENT>else:<EOL><INDENT>import glob<EOL>candidates = it.chain.from_iterable(<EOL>glob.glob(pattern) for pattern in candidates)<EOL><DEDENT>return candidates<EOL>

Search for a file or directory on your local filesystem by name
(file must be in a directory specified in a PATH environment variable)

Args:
    fname (PathLike or str): file name to match.
        If exact is False this may be a glob pattern

    path (str or Iterable[PathLike]): list of directories to search either
        specified as an os.pathsep separated string or a list of
        directories.  Defaults to environment PATH.

    exact (bool): if True, only returns exact matches. Default False.

Notes:
    For recursive behavior set `path=(d for d, _, _ in os.walk('.'))`,
    where '.' might be replaced by the root directory of interest.

Example:
    >>> list(find_path('ping', exact=True))
    >>> list(find_path('bin'))
    >>> list(find_path('bin'))
    >>> list(find_path('*cc*'))
    >>> list(find_path('cmake*'))

Example:
    >>> import ubelt as ub
    >>> from os.path import dirname
    >>> path = dirname(dirname(ub.util_platform.__file__))
    >>> res = sorted(find_path('ubelt/util_*.py', path=path))
    >>> assert len(res) >= 10
    >>> res = sorted(find_path('ubelt/util_platform.py', path=path, exact=True))
    >>> print(res)
    >>> assert len(res) == 1


from six import types<EOL>from ubelt import util_cmd<EOL>import warnings<EOL>warnings.warn('<STR_LIT>', DeprecationWarning)<EOL>if not isinstance(fpath, six.string_types):<EOL><INDENT>if isinstance(fpath, types.ModuleType):<EOL><INDENT>fpath = fpath.__file__<EOL><DEDENT>else:<EOL><INDENT>fpath =  sys.modules[fpath.__module__].__file__<EOL><DEDENT>fpath_py = fpath.replace('<STR_LIT>', '<STR_LIT>')<EOL>if exists(fpath_py):<EOL><INDENT>fpath = fpath_py<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>print('<STR_LIT>'.format(fpath))<EOL><DEDENT>editor = os.environ.get('<STR_LIT>', '<STR_LIT>')<EOL>if not exists(fpath):<EOL><INDENT>raise IOError('<STR_LIT>' % fpath)<EOL><DEDENT>util_cmd.cmd([editor, fpath], fpath, detach=True)<EOL>

DEPRICATED: This has been ported to xdev, please use that version.

Opens a file or code corresponding to a live python object in your
preferred visual editor. This function is mainly useful in an interactive
IPython session.

The visual editor is determined by the `VISUAL` environment variable.  If
this is not specified it defaults to gvim.

Args:
    fpath (PathLike): a file path or python module / function
    verbose (int): verbosity

DisableExample:
    >>> # This test interacts with a GUI frontend, not sure how to test.
    >>> import ubelt as ub
    >>> ub.editfile(ub.util_platform.__file__)
    >>> ub.editfile(ub)
    >>> ub.editfile(ub.editfile)


return platform_cache_dir()<EOL>

Alias for platform_cache_dir

DEPRICATED in favor of platform_config_dir / platform_data_dir

Returns a directory which should be writable for any application
This should be used for persistent configuration files.

Returns:
    PathLike : path to the resource dir used by the current operating system


return get_app_cache_dir(appname, *args)<EOL>

r"""
    Returns a writable directory for an application
    This should be used for persistent configuration files.

    DEPRICATED in favor of get_app_config_dir / get_app_data_dir

    Args:
        appname (str): the name of the application
        *args: any other subdirectories may be specified

    Returns:
        PathLike: dpath: writable resource directory for this application

    SeeAlso:
        ensure_app_resource_dir


return ensure_app_cache_dir(appname, *args)<EOL>

Calls `get_app_resource_dir` but ensures the directory exists.

DEPRICATED in favor of ensure_app_config_dir / ensure_app_data_dir

Args:
    appname (str): the name of the application
    *args: any other subdirectories may be specified

SeeAlso:
    get_app_resource_dir


return arg<EOL>

The identity function. Simply returns its inputs.

Args:
    arg (object): some value

Returns:
    object: arg: the same value

Example:
    >>> assert identity(42) == 42


def inject_method(self, func, name=None):

<EOL>new_method = func.__get__(self, self.__class__)<EOL>if name is None:<EOL><INDENT>name = func.__name__<EOL><DEDENT>setattr(self, name, new_method)<EOL>

Injects a function into an object instance as a bound method

The main use case of this function is for monkey patching. While monkey
patching is sometimes necessary it should generally be avoided. Thus, we
simply remind the developer that there might be a better way.

Args:
    self (object): instance to inject a function into
    func (func): the function to inject (must contain an arg for self)
    name (str): name of the method. optional. If not specified the name
        of the function is used.

Example:
    >>> class Foo(object):
    >>>     def bar(self):
    >>>         return 'bar'
    >>> def baz(self):
    >>>     return 'baz'
    >>> self = Foo()
    >>> assert self.bar() == 'bar'
    >>> assert not hasattr(self, 'baz')
    >>> inject_method(self, baz)
    >>> assert not hasattr(Foo, 'baz'), 'should only change one instance'
    >>> assert self.baz() == 'baz'
    >>> inject_method(self, baz, 'bar')
    >>> assert self.bar() == 'baz'


condensed = self._condense_cfgstr(cfgstr)<EOL>fname_cfgstr = '<STR_LIT>'.format(self.fname, condensed, self.ext)<EOL>fpath = join(self.dpath, fname_cfgstr)<EOL>fpath = normpath(fpath)<EOL>return fpath<EOL>

Reports the filepath that the cacher will use.
It will attempt to use '{fname}_{cfgstr}{ext}' unless that is too long.
Then cfgstr will be hashed.

Example:
    >>> from ubelt.util_cache import Cacher
    >>> import pytest
    >>> with pytest.warns(UserWarning):
    >>>     cacher = Cacher('test_cacher1')
    >>>     cacher.get_fpath()
    >>> self = Cacher('test_cacher2', cfgstr='cfg1')
    >>> self.get_fpath()
    >>> self = Cacher('test_cacher3', cfgstr='cfg1' * 32)
    >>> self.get_fpath()


return exists(self.get_fpath())<EOL>


import glob<EOL>pattern = join(self.dpath, self.fname + '<STR_LIT>' + self.ext)<EOL>for fname in glob.iglob(pattern):<EOL><INDENT>data_fpath = join(self.dpath, fname)<EOL>yield data_fpath<EOL><DEDENT>

Returns data with different cfgstr values that were previously computed
with this cacher.

Example:
    >>> from ubelt.util_cache import Cacher
    >>> # Ensure that some data exists
    >>> known_fnames = set()
    >>> cacher = Cacher('versioned_data', cfgstr='1')
    >>> cacher.ensure(lambda: 'data1')
    >>> known_fnames.add(cacher.get_fpath())
    >>> cacher = Cacher('versioned_data', cfgstr='2')
    >>> cacher.ensure(lambda: 'data2')
    >>> known_fnames.add(cacher.get_fpath())
    >>> # List previously computed configs for this type
    >>> from os.path import basename
    >>> cacher = Cacher('versioned_data', cfgstr='2')
    >>> exist_fpaths = set(cacher.existing_versions())
    >>> exist_fnames = list(map(basename, exist_fpaths))
    >>> print(exist_fnames)
    >>> assert exist_fpaths == known_fnames

    ['versioned_data_1.pkl', 'versioned_data_2.pkl']


data_fpath = self.get_fpath(cfgstr)<EOL>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>')<EOL><DEDENT>if exists(data_fpath):<EOL><INDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>'.format(data_fpath))<EOL><DEDENT>os.remove(data_fpath)<EOL>meta_fpath = data_fpath + '<STR_LIT>'<EOL>if exists(meta_fpath):<EOL><INDENT>os.remove(meta_fpath)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>')<EOL><DEDENT><DEDENT>

Removes the saved cache and metadata from disk


def tryload(self, cfgstr=None, on_error='<STR_LIT>'):

cfgstr = self._rectify_cfgstr(cfgstr)<EOL>if self.enabled:<EOL><INDENT>try:<EOL><INDENT>if self.verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT>return self.load(cfgstr)<EOL><DEDENT>except IOError:<EOL><INDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>')<EOL><DEDENT>if on_error == '<STR_LIT>':<EOL><INDENT>raise<EOL><DEDENT>elif on_error == '<STR_LIT>':<EOL><INDENT>self.clear(cfgstr)<EOL>return None<EOL><DEDENT>else:<EOL><INDENT>raise KeyError('<STR_LIT>'.format(on_error))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if self.verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT><DEDENT>return None<EOL>

Like load, but returns None if the load fails due to a cache miss.

Args:
    on_error (str): How to handle non-io errors errors. Either raise,
        which re-raises the exception, or clear which deletes the cache
        and returns None.


from six.moves import cPickle as pickle<EOL>cfgstr = self._rectify_cfgstr(cfgstr)<EOL>dpath = self.dpath<EOL>fname = self.fname<EOL>verbose = self.verbose<EOL>if not self.enabled:<EOL><INDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT>raise IOError(<NUM_LIT:3>, '<STR_LIT>')<EOL><DEDENT>fpath = self.get_fpath(cfgstr=cfgstr)<EOL>if not exists(fpath):<EOL><INDENT>if verbose > <NUM_LIT:2>:<EOL><INDENT>self.log('<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>basename(dpath), fname, cfgstr))<EOL><DEDENT>raise IOError(<NUM_LIT:2>, '<STR_LIT>' % (fpath,))<EOL><DEDENT>else:<EOL><INDENT>if verbose > <NUM_LIT:3>:<EOL><INDENT>self.log('<STR_LIT>'<EOL>'<STR_LIT>'.format(<EOL>basename(dpath), fname, cfgstr))<EOL><DEDENT><DEDENT>try:<EOL><INDENT>with open(fpath, '<STR_LIT:rb>') as file_:<EOL><INDENT>data = pickle.load(file_)<EOL><DEDENT><DEDENT>except Exception as ex:<EOL><INDENT>if verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>' % (fpath,))<EOL><DEDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>'.format(<EOL>basename(dpath), cfgstr))<EOL><DEDENT>if isinstance(ex, (EOFError, IOError, ImportError)):<EOL><INDENT>raise IOError(str(ex))<EOL><DEDENT>else:<EOL><INDENT>if verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>')<EOL><DEDENT>raise<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.verbose > <NUM_LIT:2>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT>elif verbose > <NUM_LIT:1>:<EOL><INDENT>self.log('<STR_LIT>')<EOL><DEDENT><DEDENT>return data<EOL>

Loads the data

Raises:
    IOError - if the data is unable to be loaded. This could be due to
        a cache miss or because the cache is disabled.

Example:
    >>> from ubelt.util_cache import *  # NOQA
    >>> # Setting the cacher as enabled=False turns it off
    >>> cacher = Cacher('test_disabled_load', '', enabled=True)
    >>> cacher.save('data')
    >>> assert cacher.load() == 'data'
    >>> cacher.enabled = False
    >>> assert cacher.tryload() is None


from six.moves import cPickle as pickle<EOL>if not self.enabled:<EOL><INDENT>return<EOL><DEDENT>if self.verbose > <NUM_LIT:0>:<EOL><INDENT>self.log('<STR_LIT>'.format(self.fname))<EOL><DEDENT>cfgstr = self._rectify_cfgstr(cfgstr)<EOL>condensed = self._condense_cfgstr(cfgstr)<EOL>util_path.ensuredir(self.dpath)<EOL>data_fpath = self.get_fpath(cfgstr=cfgstr)<EOL>meta_fpath = data_fpath + '<STR_LIT>'<EOL>with open(meta_fpath, '<STR_LIT:a>') as file_:<EOL><INDENT>file_.write('<STR_LIT>'.format(util_time.timestamp()))<EOL>file_.write(self.fname + '<STR_LIT:\n>')<EOL>file_.write(condensed + '<STR_LIT:\n>')<EOL>file_.write(cfgstr + '<STR_LIT:\n>')<EOL>file_.write(str(self.meta) + '<STR_LIT:\n>')<EOL><DEDENT>with open(data_fpath, '<STR_LIT:wb>') as file_:<EOL><INDENT>pickle.dump(data, file_, protocol=self.protocol)<EOL><DEDENT>

Writes data to path specified by `self.fpath(cfgstr)`.

Metadata containing information about the cache will also be appended
to an adjacent file with the `.meta` suffix.

Example:
    >>> from ubelt.util_cache import *  # NOQA
    >>> # Normal functioning
    >>> cfgstr = 'long-cfg' * 32
    >>> cacher = Cacher('test_enabled_save', cfgstr)
    >>> cacher.save('data')
    >>> assert exists(cacher.get_fpath()), 'should be enabeled'
    >>> assert exists(cacher.get_fpath() + '.meta'), 'missing metadata'
    >>> # Setting the cacher as enabled=False turns it off
    >>> cacher2 = Cacher('test_disabled_save', 'params', enabled=False)
    >>> cacher2.save('data')
    >>> assert not exists(cacher2.get_fpath()), 'should be disabled'


def ensure(self, func, *args, **kwargs):

data = self.tryload()<EOL>if data is None:<EOL><INDENT>data = func(*args, **kwargs)<EOL>self.save(data)<EOL><DEDENT>return data<EOL>

r"""
        Wraps around a function. A cfgstr must be stored in the base cacher.

        Args:
            func (callable): function that will compute data on cache miss
            *args: passed to func
            **kwargs: passed to func

        Example:
            >>> from ubelt.util_cache import *  # NOQA
            >>> def func():
            >>>     return 'expensive result'
            >>> fname = 'test_cacher_ensure'
            >>> cfgstr = 'func params'
            >>> cacher = Cacher(fname, cfgstr)
            >>> cacher.clear()
            >>> data1 = cacher.ensure(func)
            >>> data2 = cacher.ensure(func)
            >>> assert data1 == 'expensive result'
            >>> assert data1 == data2
            >>> cacher.clear()


<EOL>def _wrapper():<EOL><INDENT>data = self.ensure(func)<EOL>return data<EOL><DEDENT>_wrapper.cacher = self<EOL>return _wrapper<EOL>

Allows Cacher to be used as a decorator for functions with no
arguments. This mode of usage has much less control than others, so it
is only recommended for the simplest of cases.

Args:
    func (Callable): function to decorate. Must have no arguments.

Example:
    >>> from ubelt.util_cache import *  # NOQA
    >>> @Cacher('demo_cacher_call', cfgstr='foobar')
    >>> def func():
    >>>     return 'expensive result'
    >>> func.cacher.clear()
    >>> assert not func.cacher.exists()
    >>> data = func()
    >>> assert func.cacher.exists()
    >>> func.cacher.clear()


certificate = self.cacher.tryload(cfgstr=cfgstr)<EOL>return certificate<EOL>

Returns the stamp certificate if it exists


products = self.product if product is None else product<EOL>if products is None:<EOL><INDENT>return None<EOL><DEDENT>if not isinstance(products, (list, tuple)):<EOL><INDENT>products = [products]<EOL><DEDENT>return products<EOL>


if self.hasher is None:<EOL><INDENT>return None<EOL><DEDENT>else:<EOL><INDENT>products = self._rectify_products(product)<EOL>product_file_hash = [<EOL>util_hash.hash_file(p, hasher=self.hasher, base='<STR_LIT>')<EOL>for p in products<EOL>]<EOL>return product_file_hash<EOL><DEDENT>


def expired(self, cfgstr=None, product=None):

products = self._rectify_products(product)<EOL>certificate = self._get_certificate(cfgstr=cfgstr)<EOL>if certificate is None:<EOL><INDENT>is_expired = True<EOL><DEDENT>elif products is None:<EOL><INDENT>is_expired = False<EOL><DEDENT>elif not all(map(os.path.exists, products)):<EOL><INDENT>is_expired = True<EOL><DEDENT>else:<EOL><INDENT>product_file_hash = self._product_file_hash(products)<EOL>certificate_hash = certificate.get('<STR_LIT>', None)<EOL>is_expired = product_file_hash != certificate_hash<EOL><DEDENT>return is_expired<EOL>

Check to see if a previously existing stamp is still valid and if the
expected result of that computation still exists.

Args:
    cfgstr (str, optional): override the default cfgstr if specified
    product (PathLike or Sequence[PathLike], optional): override the
        default product if specified


def renew(self, cfgstr=None, product=None):

products = self._rectify_products(product)<EOL>certificate = {<EOL>'<STR_LIT>': util_time.timestamp(),<EOL>'<STR_LIT>': products,<EOL>}<EOL>if products is not None:<EOL><INDENT>if not all(map(os.path.exists, products)):<EOL><INDENT>raise IOError(<EOL>'<STR_LIT>'.format(products))<EOL><DEDENT>certificate['<STR_LIT>'] = self._product_file_hash(products)<EOL><DEDENT>self.cacher.save(certificate, cfgstr=cfgstr)<EOL>return certificate<EOL>

Recertify that the product has been recomputed by writing a new
certificate to disk.


def make_network_graph(compact, expression_names, lookup_names):

nodes = set()<EOL>edges = defaultdict(set) <EOL>var_name_dict = dict()<EOL>if expression_names:<EOL><INDENT>for e in graphviz_items: <EOL><INDENT>if e in expression_names:<EOL><INDENT>var_name_dict[e.vindex] = expression_names[e]<EOL><DEDENT><DEDENT><DEDENT>rnn_bldr_name = defaultdict(lambda: chr(len(rnn_bldr_name)+ord('<STR_LIT:A>')))<EOL>def vidx2str(vidx): return '<STR_LIT>' % ('<STR_LIT:N>', vidx)<EOL>for e in graphviz_items: <EOL><INDENT>vidx = e.vindex<EOL>f_name = e.name<EOL>args = e.args<EOL>output_dim = e.dim<EOL>input_dim = None <EOL>children = set()<EOL>node_type = '<STR_LIT>'<EOL>if f_name == '<STR_LIT>':<EOL><INDENT>[_dim] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_v] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_d1, _d2] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_v, _d] = args<EOL>arg_strs = []<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_dim] = args<EOL>arg_strs = []<EOL>if compact:<EOL><INDENT>if vidx in var_name_dict:<EOL><INDENT>f_name = var_name_dict[vidx]<EOL><DEDENT><DEDENT>node_type = '<STR_LIT>'<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[_dim] = args<EOL>arg_strs = []<EOL>if compact:<EOL><INDENT>if vidx in var_name_dict:<EOL><INDENT>f_name = var_name_dict[vidx]<EOL><DEDENT><DEDENT>node_type = '<STR_LIT>'<EOL><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[p, idx, update] = args<EOL>[_dim] = p.args<EOL>if vidx in var_name_dict:<EOL><INDENT>name = var_name_dict[vidx]<EOL><DEDENT>else:<EOL><INDENT>name = None<EOL><DEDENT>item_name = None<EOL>if lookup_names and p in expression_names:<EOL><INDENT>param_name = expression_names[p]<EOL>if param_name in lookup_names:<EOL><INDENT>item_name = '<STR_LIT>' % (lookup_names[param_name][idx],)<EOL><DEDENT><DEDENT>if compact:<EOL><INDENT>if item_name is not None:<EOL><INDENT>f_name = item_name<EOL><DEDENT>elif name is not None:<EOL><INDENT>f_name = '<STR_LIT>' % (name, idx)<EOL><DEDENT>else:<EOL><INDENT>f_name = '<STR_LIT>' % (idx)<EOL><DEDENT>arg_strs = []<EOL><DEDENT>else:<EOL><INDENT>arg_strs = [var_name_dict.get(p.vindex, '<STR_LIT>' % (p.vindex))]<EOL>if item_name is not None:<EOL><INDENT>arg_strs.append(item_name)<EOL><DEDENT>vocab_size = _dim[<NUM_LIT:0>]<EOL>arg_strs.extend(['<STR_LIT:%s>' % (idx), '<STR_LIT:%s>' % (vocab_size), '<STR_LIT>' if update else '<STR_LIT>'])<EOL><DEDENT><DEDENT>elif f_name == '<STR_LIT>':<EOL><INDENT>[arg, input_dim, bldr_type, bldr_num, state_idx] = args <EOL>rnn_name = rnn_bldr_name[bldr_num]<EOL>if bldr_type.endswith('<STR_LIT>'):<EOL><INDENT>bldr_type[:-len('<STR_LIT>')]<EOL><DEDENT>f_name = '<STR_LIT>' % (bldr_type, rnn_name, state_idx)<EOL>if not compact:<EOL><INDENT>i = arg.vindex<EOL>s = var_name_dict.get(i, '<STR_LIT>' % (i))<EOL>arg_strs = [s]<EOL><DEDENT>else:<EOL><INDENT>arg_strs = []<EOL><DEDENT>children.add(vidx2str(arg.vindex))<EOL>node_type = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>arg_strs = []<EOL>for arg in args:<EOL><INDENT>if isinstance(arg, Expression):<EOL><INDENT>if not compact:<EOL><INDENT>i = arg.vindex<EOL>s = var_name_dict.get(i, '<STR_LIT>' % (i))<EOL>arg_strs.append(s)<EOL><DEDENT>children.add(vidx2str(arg.vindex))<EOL><DEDENT>elif isinstance(arg, float) and compact:<EOL><INDENT>s = re.sub('<STR_LIT>', '<STR_LIT>', '<STR_LIT>' % (arg))<EOL>if s == '<STR_LIT>':<EOL><INDENT>s = str(arg)<EOL><DEDENT>arg_strs.append(s)<EOL><DEDENT>else:<EOL><INDENT>arg_strs.append(str(arg))<EOL>

Make a network graph, represented as of nodes and a set of edges.
  The nodes are represented as tuples: (name: string, input_dim: Dim, label: string, output_dim: Dim, children: set[name], features: string)
#   The edges are represented as dict of children to sets of parents: (child: string) -> [(parent: string, features: string)]


def __init__(self, num_layers, input_dim, hidden_dim, model, rnn_builder_factory):

assert num_layers > <NUM_LIT:0><EOL>assert hidden_dim % <NUM_LIT:2> == <NUM_LIT:0><EOL>self.builder_layers = []<EOL>f = rnn_builder_factory(<NUM_LIT:1>, input_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>b = rnn_builder_factory(<NUM_LIT:1>, input_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>self.builder_layers.append((f,b))<EOL>for _ in xrange(num_layers-<NUM_LIT:1>):<EOL><INDENT>f = rnn_builder_factory(<NUM_LIT:1>, hidden_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>b = rnn_builder_factory(<NUM_LIT:1>, hidden_dim, hidden_dim/<NUM_LIT:2>, model)<EOL>self.builder_layers.append((f,b))<EOL><DEDENT>

@param num_layers: depth of the BiRNN
@param input_dim: size of the inputs
@param hidden_dim: size of the outputs (and intermediate layer representations)
@param model
@param rnn_builder_factory: RNNBuilder subclass, e.g. LSTMBuilder


for e in es:<EOL><INDENT>ensure_freshness(e)<EOL><DEDENT>for (fb,bb) in self.builder_layers[:-<NUM_LIT:1>]:<EOL><INDENT>fs = fb.initial_state().transduce(es)<EOL>bs = bb.initial_state().transduce(reversed(es))<EOL>es = [concatenate([f,b]) for f,b in zip(fs, reversed(bs))]<EOL><DEDENT>(fb,bb) = self.builder_layers[-<NUM_LIT:1>]<EOL>fs = fb.initial_state().add_inputs(es)<EOL>bs = bb.initial_state().add_inputs(reversed(es))<EOL>return [(f,b) for f,b in zip(fs, reversed(bs))]<EOL>

returns the list of state pairs (stateF, stateB) obtained by adding
inputs to both forward (stateF) and backward (stateB) RNNs.

@param es: a list of Expression

see also transduce(xs)

.transduce(xs) is different from .add_inputs(xs) in the following way:

    .add_inputs(xs) returns a list of RNNState pairs. RNNState objects can be
     queried in various ways. In particular, they allow access to the previous
     state, as well as to the state-vectors (h() and s() )

    .transduce(xs) returns a list of Expression. These are just the output
     expressions. For many cases, this suffices.
     transduce is much more memory efficient than add_inputs.


for e in es:<EOL><INDENT>ensure_freshness(e)<EOL><DEDENT>for (fb,bb) in self.builder_layers:<EOL><INDENT>fs = fb.initial_state().transduce(es)<EOL>bs = bb.initial_state().transduce(reversed(es))<EOL>es = [concatenate([f,b]) for f,b in zip(fs, reversed(bs))]<EOL><DEDENT>return es<EOL>

returns the list of output Expressions obtained by adding the given inputs
to the current state, one by one, to both the forward and backward RNNs,
and concatenating.

@param es: a list of Expression

see also add_inputs(xs)

.transduce(xs) is different from .add_inputs(xs) in the following way:

    .add_inputs(xs) returns a list of RNNState pairs. RNNState objects can be
     queried in various ways. In particular, they allow access to the previous
     state, as well as to the state-vectors (h() and s() )

    .transduce(xs) returns a list of Expression. These are just the output
     expressions. For many cases, this suffices.
     transduce is much more memory efficient than add_inputs.


states = []<EOL>cur = self<EOL>for x in xs:<EOL><INDENT>cur = cur.add_input(x)<EOL>states.append(cur)<EOL><DEDENT>return states<EOL>

returns the list of states obtained by adding the given inputs
to the current state, one by one.


out_args = []<EOL>if arg_str is None:<EOL><INDENT>return out_str<EOL><DEDENT>args = arg_str.split('<STR_LIT:U+002C>')<EOL>for arg in args:<EOL><INDENT>components = arg.split('<STR_LIT:=>')<EOL>name_and_type=components[<NUM_LIT:0>].split('<STR_LIT:U+0020>')<EOL>if name_and_type[-<NUM_LIT:1>]=='<STR_LIT>' and len(name_and_type)><NUM_LIT:1>:<EOL><INDENT>name=name_and_type[-<NUM_LIT:2>]<EOL><DEDENT>else:<EOL><INDENT>name=name_and_type[-<NUM_LIT:1>]<EOL><DEDENT>if len(components)><NUM_LIT:1>:<EOL><INDENT>name+='<STR_LIT:=>'+components[<NUM_LIT:1>]<EOL><DEDENT>out_args.append(name)<EOL><DEDENT>return '<STR_LIT:U+002C>'.join(out_args)<EOL>

Remove types from function arguments in cython


if indent_str is None:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return indent_str<EOL><DEDENT>


if inherit_str is None:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>return inherit_str<EOL><DEDENT>


name = func_str.split('<STR_LIT:U+0020>')[-<NUM_LIT:1>]<EOL>return name<EOL>

Get function name, ie removes possible return type


dest_directory = '<STR_LIT:.>'<EOL>filename = DATA_URL.split('<STR_LIT:/>')[-<NUM_LIT:1>]<EOL>filepath = os.path.join(dest_directory, filename)<EOL>if not os.path.exists(filepath):<EOL><INDENT>def _progress(count, block_size, total_size):<EOL><INDENT>sys.stdout.write('<STR_LIT>' % (filename,<EOL>float(count * block_size) / float(total_size) * <NUM_LIT>))<EOL>sys.stdout.flush()<EOL><DEDENT>filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)<EOL>print()<EOL>statinfo = os.stat(filepath)<EOL>print('<STR_LIT>', filename, statinfo.st_size, '<STR_LIT>')<EOL><DEDENT>extracted_dir_path = os.path.join(dest_directory, '<STR_LIT>')<EOL>if not os.path.exists(extracted_dir_path):<EOL><INDENT>zip_ref = zipfile.ZipFile(filepath, '<STR_LIT:r>')<EOL>zip_ref.extractall(dest_directory)<EOL>zip_ref.close()<EOL><DEDENT>

Download and extract processed data and embeddings.


if dataset is "<STR_LIT>":<EOL><INDENT>fname_img = os.path.join(path, "<STR_LIT>")<EOL>fname_lbl = os.path.join(path, "<STR_LIT>")<EOL><DEDENT>elif dataset is "<STR_LIT>":<EOL><INDENT>fname_img = os.path.join(path, "<STR_LIT>")<EOL>fname_lbl = os.path.join(path, "<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>with open(fname_lbl, "<STR_LIT:rb>") as flbl:<EOL><INDENT>magic, num = struct.unpack("<STR_LIT>", flbl.read(<NUM_LIT:8>))<EOL>labels = np.fromfile(flbl, dtype=np.int8)<EOL><DEDENT>with open(fname_img, "<STR_LIT:rb>") as fimg:<EOL><INDENT>magic, num, rows, cols = struct.unpack("<STR_LIT>", fimg.read(<NUM_LIT:16>))<EOL>images = np.multiply(<EOL>np.fromfile(fimg, dtype=np.uint8).reshape(len(labels), rows*cols),<EOL><NUM_LIT:1.0> / <NUM_LIT>)<EOL><DEDENT>get_instance = lambda idx: (labels[idx], images[idx].reshape(<NUM_LIT:1>, <NUM_LIT>, <NUM_LIT>))<EOL>size_reset = lambda x: x.reshape(<NUM_LIT:1>, <NUM_LIT>, <NUM_LIT>)<EOL>return list(map(size_reset, images))<EOL>

wget -O - http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz | gunzip > train-images-idx3-ubyte
wget -O - http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz | gunzip > train-labels-idx1-ubyte
wget -O - http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz | gunzip > t10k-images-idx3-ubyte
wget -O - http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz | gunzip > t10k-labels-idx1-ubyte


def make_grid(tensor, nrow=<NUM_LIT:8>, padding=<NUM_LIT:2>, pad_value=<NUM_LIT:0>):

if not (isinstance(tensor, np.ndarray) or<EOL>(isinstance(tensor, list) and all(isinstance(t, np.ndarray) for t in tensor))):<EOL><INDENT>raise TypeError('<STR_LIT>'.format(type(tensor)))<EOL><DEDENT>if isinstance(tensor, list):<EOL><INDENT>tensor = np.stack(tensor, <NUM_LIT:0>)<EOL><DEDENT>if tensor.ndim == <NUM_LIT:2>:  <EOL><INDENT>tensor = tensor.reshape((<NUM_LIT:1>, tensor.shape[<NUM_LIT:0>], tensor.shape[<NUM_LIT:1>]))<EOL><DEDENT>if tensor.ndim == <NUM_LIT:3>:<EOL><INDENT>if tensor.shape[<NUM_LIT:0>] == <NUM_LIT:1>:  <EOL><INDENT>tensor = np.concatenate((tensor, tensor, tensor), <NUM_LIT:0>)<EOL><DEDENT>tensor = tensor.reshape((<NUM_LIT:1>, tensor.shape[<NUM_LIT:0>], tensor.shape[<NUM_LIT:1>], tensor.shape[<NUM_LIT:2>]))<EOL><DEDENT>if tensor.ndim == <NUM_LIT:4> and tensor.shape[<NUM_LIT:1>] == <NUM_LIT:1>:  <EOL><INDENT>tensor = np.concatenate((tensor, tensor, tensor), <NUM_LIT:1>)<EOL><DEDENT>if tensor.shape[<NUM_LIT:0>] == <NUM_LIT:1>:<EOL><INDENT>return np.squeeze(tensor)<EOL><DEDENT>nmaps = tensor.shape[<NUM_LIT:0>]<EOL>xmaps = min(nrow, nmaps)<EOL>ymaps = int(math.ceil(float(nmaps) / xmaps))<EOL>height, width = int(tensor.shape[<NUM_LIT:2>] + padding), int(tensor.shape[<NUM_LIT:3>] + padding)<EOL>grid = np.ones((<NUM_LIT:3>, height * ymaps + padding, width * xmaps + padding)) * pad_value<EOL>k = <NUM_LIT:0><EOL>for y in range(ymaps):<EOL><INDENT>for x in range(xmaps):<EOL><INDENT>if k >= nmaps:<EOL><INDENT>break<EOL><DEDENT>grid[:, y * height + padding:(y+<NUM_LIT:1>) * height,x * width + padding:(x+<NUM_LIT:1>) * width] = tensor[k]<EOL>k = k + <NUM_LIT:1><EOL><DEDENT><DEDENT>return grid<EOL>

Make a grid of images, via numpy.

    Args:
        tensor (Tensor or list): 4D mini-batch Tensor of shape (B x C x H x W)
            or a list of images all of the same size.
        nrow (int, optional): Number of images displayed in each row of the grid.
            The Final grid size is (B / nrow, nrow). Default is 8.
        padding (int, optional): amount of padding. Default is 2.
        pad_value (float, optional): Value for the padded pixels.


def save_image(tensor, filename, nrow=<NUM_LIT:8>, padding=<NUM_LIT:2>, pad_value=<NUM_LIT:0>):

from PIL import Image<EOL>grid = make_grid(tensor, nrow=nrow, padding=padding, pad_value=pad_value)<EOL>im = Image.fromarray(pre_pillow_float_img_process(grid))<EOL>im.save(filename)<EOL>

Save a given Tensor into an image file.

    Args:
        tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,
            saves the tensor as a grid of images by calling ``make_grid``.
        **kwargs: Other arguments are documented in ``make_grid``.


<EOL>if not hasattr(get_tools, '<STR_LIT>'):<EOL><INDENT>get_tools._CACHE = dict()<EOL>for entry in pkg_resources.iter_entry_points('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>get_tools._CACHE[entry.name] = entry.load()<EOL><DEDENT>except ImportError as exc:  <EOL><INDENT>output_error(<EOL>'<STR_LIT>' % (<EOL>entry,<EOL>entry.dist,<EOL>exc,<EOL>),<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return get_tools._CACHE<EOL>

Retrieves the TidyPy tools that are available in the current Python
environment.

The returned dictionary has keys that are the tool names and values are the
tool classes.

:rtype: dict


<EOL>if not hasattr(get_reports, '<STR_LIT>'):<EOL><INDENT>get_reports._CACHE = dict()<EOL>for entry in pkg_resources.iter_entry_points('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>get_reports._CACHE[entry.name] = entry.load()<EOL><DEDENT>except ImportError as exc:  <EOL><INDENT>output_error(<EOL>'<STR_LIT>' % (<EOL>entry,<EOL>entry.dist,<EOL>exc,<EOL>),<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return get_reports._CACHE<EOL>

Retrieves the TidyPy issue reports that are available in the current Python
environment.

The returned dictionary has keys are the report names and values are the
report classes.

:rtype: dict


<EOL>if not hasattr(get_extenders, '<STR_LIT>'):<EOL><INDENT>get_extenders._CACHE = dict()<EOL>for entry in pkg_resources.iter_entry_points('<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>get_extenders._CACHE[entry.name] = entry.load()<EOL><DEDENT>except ImportError as exc:  <EOL><INDENT>output_error(<EOL>'<STR_LIT>' % (<EOL>entry,<EOL>entry.dist,<EOL>exc,<EOL>),<EOL>)<EOL><DEDENT><DEDENT><DEDENT>return get_extenders._CACHE<EOL>

Retrieves the TidyPy configuration extenders that are available in the
current Python environment.

The returned dictionary has keys are the extender names and values are the
extender classes.

:rtype: dict


cache_path = get_cache_path(location)<EOL>if location:<EOL><INDENT>os.remove(cache_path)<EOL><DEDENT>else:<EOL><INDENT>shutil.rmtree(cache_path)<EOL><DEDENT>

Clears out the cache of TidyPy configurations that were retrieved from
outside the normal locations.


config = {}<EOL>for name, cls in iteritems(get_tools()):<EOL><INDENT>config[name] = cls.get_default_config()<EOL><DEDENT>try:<EOL><INDENT>workers = multiprocessing.cpu_count() - <NUM_LIT:1><EOL><DEDENT>except NotImplementedError:  <EOL><INDENT>workers = <NUM_LIT:1><EOL><DEDENT>workers = max(<NUM_LIT:1>, min(<NUM_LIT:4>, workers))<EOL>config.update({<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': workers,<EOL>'<STR_LIT>': [<EOL>{<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>},<EOL>],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': False,<EOL>})<EOL>return config<EOL>

Produces a stock/out-of-the-box TidyPy configuration.

:rtype: dict


if sys.platform == '<STR_LIT:win32>':<EOL><INDENT>user_config = os.path.expanduser(r'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>user_config = os.path.join(<EOL>os.getenv('<STR_LIT>') or os.path.expanduser('<STR_LIT>'),<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>if os.path.exists(user_config):<EOL><INDENT>with open(user_config, '<STR_LIT:r>') as config_file:<EOL><INDENT>config = pytoml.load(config_file).get('<STR_LIT>', {})<EOL><DEDENT>config = merge_dict(get_default_config(), config)<EOL>config = process_extensions(config, project_path, use_cache=use_cache)<EOL>return config<EOL><DEDENT>return None<EOL>

Produces a TidyPy configuration that incorporates the configuration files
stored in the current user's home directory.

:param project_path: the path to the project that is going to be analyzed
:type project_path: str
:param use_cache:
    whether or not to use cached versions of any remote/referenced TidyPy
    configurations. If not specified, defaults to ``True``.
:type use_cache: bool
:rtype: dict


pyproject_path = os.path.join(project_path, '<STR_LIT>')<EOL>if os.path.exists(pyproject_path):<EOL><INDENT>with open(pyproject_path, '<STR_LIT:r>') as config_file:<EOL><INDENT>config = pytoml.load(config_file)<EOL><DEDENT>config = config.get('<STR_LIT>', {}).get('<STR_LIT>', {})<EOL>config = merge_dict(get_default_config(), config)<EOL>config = process_extensions(config, project_path, use_cache=use_cache)<EOL>return config<EOL><DEDENT>return None<EOL>

Produces a TidyPy configuration using the ``pyproject.toml`` in the
project's directory.

:param project_path: the path to the project that is going to be analyzed
:type project_path: str
:param use_cache:
    whether or not to use cached versions of any remote/referenced TidyPy
    configurations. If not specified, defaults to ``True``.
:type use_cache: bool
:rtype: dict


return get_local_config(project_path, use_cache=use_cache)or get_user_config(project_path, use_cache=use_cache)or get_default_config()<EOL>

Produces the Tidypy configuration to use for the specified project.

If a ``pyproject.toml`` exists, the configuration will be based on that. If
not, the TidyPy configuration in the user's home directory will be used. If
one does not exist, the default configuration will be used.

:param project_path: the path to the project that is going to be analyzed
:type project_path: str
:param use_cache:
    whether or not to use cached versions of any remote/referenced TidyPy
    configurations. If not specified, defaults to ``True``.
:type use_cache: bool
:rtype: dict


merged = list(list1)<EOL>for value in list2:<EOL><INDENT>if value not in merged:<EOL><INDENT>merged.append(value)<EOL><DEDENT><DEDENT>return merged<EOL>

Merges the contents of two lists into a new list.

:param list1: the first list
:type list1: list
:param list2: the second list
:type list2: list
:returns: list


def merge_dict(dict1, dict2, merge_lists=False):

merged = dict(dict1)<EOL>for key, value in iteritems(dict2):<EOL><INDENT>if isinstance(merged.get(key), dict):<EOL><INDENT>merged[key] = merge_dict(merged[key], value)<EOL><DEDENT>elif merge_lists and isinstance(merged.get(key), list):<EOL><INDENT>merged[key] = merge_list(merged[key], value)<EOL><DEDENT>else:<EOL><INDENT>merged[key] = value<EOL><DEDENT><DEDENT>return merged<EOL>

Recursively merges the contents of two dictionaries into a new dictionary.

When both input dictionaries share a key, the value from ``dict2`` is
kept.

:param dict1: the first dictionary
:type dict1: dict
:param dict2: the second dictionary
:type dict2: dict
:param merge_lists:
    when this function encounters a key that contains lists in both input
    dictionaries, this parameter dictates whether or not those lists should
    be merged. If not specified, defaults to ``False``.
:type merge_lists: bool
:returns: dict


click.echo(click.style(msg, fg='<STR_LIT>'), err=True)<EOL>

Prints the specified string to ``stderr``.

:param msg: the message to print
:type msg: str


old_path = sys.path<EOL>sys.path = paths + sys.path<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>sys.path = old_path<EOL><DEDENT>

A context manager that will append the specified paths to Python's
``sys.path`` during the execution of the block.

:param paths: the paths to append
:type paths: list(str)


if not masks:<EOL><INDENT>masks = []<EOL><DEDENT>elif not isinstance(masks, (list, tuple)):<EOL><INDENT>masks = [masks]<EOL><DEDENT>return [<EOL>re.compile(mask)<EOL>for mask in masks<EOL>]<EOL>

Compiles a list of regular expressions.

:param masks: the regular expressions to compile
:type masks: list(str) or str
:returns: list(regular expression object)


for mask in masks:<EOL><INDENT>if mask.search(target):<EOL><INDENT>return True<EOL><DEDENT><DEDENT>return False<EOL>

Determines whether or not the target string matches any of the regular
expressions specified.

:param target: the string to check
:type target: str
:param masks: the regular expressions to check against
:type masks: list(regular expression object)
:returns: bool


with _FILE_CACHE_LOCK:<EOL><INDENT>if filepath not in _FILE_CACHE:<EOL><INDENT>_FILE_CACHE[filepath] = _read_file(filepath)<EOL><DEDENT><DEDENT>return _FILE_CACHE[filepath]<EOL>

Retrieves the contents of the specified file.

This function performs simple caching so that the same file isn't read more
than once per process.

:param filepath: the file to read
:type filepath: str
:returns: str


with _AST_CACHE_LOCK:<EOL><INDENT>if filepath not in _AST_CACHE:<EOL><INDENT>source = read_file(filepath)<EOL>_AST_CACHE[filepath] = ast.parse(source, filename=filepath)<EOL><DEDENT><DEDENT>return _AST_CACHE[filepath]<EOL>

Retrieves the AST of the specified file.

This function performs simple caching so that the same file isn't read or
parsed more than once per process.

:param filepath: the file to parse
:type filepath: str
:returns: ast.AST


return _REQUESTS<EOL>

Retrieves a ``requests`` object to use within TidyPy.


return self._stdout.getvalue()<EOL>

Retrieves the content that was written to ``stdout``.

:returns: str


return self._stderr.getvalue()<EOL>

Retrieves the content that was written to ``stderr``.

:returns: str


return True<EOL>

Indicates whether or not this tool can be executed now. Useful when you
need to check for certain environmental conditions (e.g., Python
version, dependency availability, etc).

Unless overridden, always returns ``True``.

:rtype: bool


return {<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': [],<EOL>'<STR_LIT>': {},<EOL>}<EOL>

Produces a tool configuration stanza that acts as the base/default for
this tool.

rtype: dict


raise NotImplementedError()<EOL>

Produces a sequence of all the issue codes this tool is capable of
generating. Elements in this sequence must all be 2-element tuples,
where the first element is the code, and the second is a textual
description of what the code means.

Must be implemented by concrete classes.

:returns: tuple of tuples containing two strings each


<EOL>self.config = config<EOL>

:param config: the tool configuration to use during execution
:type config: dict


raise NotImplementedError()<EOL>

Analyzes the project and generates a list of issues found during that
analysis.

Must be implemented by concrete classes.

:param finder:
    the Finder class that should be used to identify the files or
    directories that the tool will analyze.
:type finder: tidypy.Finder
:rtype: list(tidypy.Issue)


Called when the execution of the TidyPy tool suite begins.


with self._lock:<EOL><INDENT>self.current_tools.append(tool)<EOL><DEDENT>

Called when an individual tool begins execution.

:param tool: the name of the tool that is starting
:type tool: str


with self._lock:<EOL><INDENT>if tool in self.current_tools:<EOL><INDENT>self.current_tools.remove(tool)<EOL>self.completed_tools.append(tool)<EOL><DEDENT><DEDENT>

Called when an individual tool completes execution.

:param tool: the name of the tool that completed
:type tool: str


Called after all tools in the suite have completed.


self._config = config<EOL>self._all_issues = []<EOL>self._cleaned_issues = None<EOL>self._lock = Lock()<EOL>self._noqa = {}<EOL>

:param config:
    the configuration used to during the analysis of the project
:type config: dict


if not isinstance(issues, (list, tuple)):<EOL><INDENT>issues = [issues]<EOL><DEDENT>with self._lock:<EOL><INDENT>self._all_issues.extend(issues)<EOL>self._cleaned_issues = None<EOL><DEDENT>

Adds an issue to the collection.

:param issues: the issue(s) to add
:type issues: tidypy.Issue or list(tidypy.Issue)


if include_unclean:<EOL><INDENT>return len(self._all_issues)<EOL><DEDENT>self._ensure_cleaned_issues()<EOL>return len(self._cleaned_issues)<EOL>

Returns the number of issues in the collection.

:param include_unclean:
    whether or not to include issues that are being ignored due to
    being a duplicate, excluded, etc.
:type include_unclean: bool
:rtype: int


self._ensure_cleaned_issues()<EOL>return self._sort_issues(self._cleaned_issues, sortby)<EOL>

Retrieves the issues in the collection.

:param sortby: the properties to sort the issues by
:type sortby: list(str)
:rtype: list(tidypy.Issue)


def get_grouped_issues(self, keyfunc=None, sortby=None):

if not keyfunc:<EOL><INDENT>keyfunc = default_group<EOL><DEDENT>if not sortby:<EOL><INDENT>sortby = self.DEFAULT_SORT<EOL><DEDENT>self._ensure_cleaned_issues()<EOL>return self._group_issues(self._cleaned_issues, keyfunc, sortby)<EOL>

Retrieves the issues in the collection grouped into buckets according
to the key generated by the keyfunc.

:param keyfunc:
    a function that will be used to generate the key that identifies
    the group that an issue will be assigned to. This function receives
    a single tidypy.Issue argument and must return a string. If not
    specified, the filename of the issue will be used.
:type keyfunc: func
:param sortby: the properties to sort the issues by
:type sortby: list(str)
:rtype: OrderedDict


raise NotImplementedError()<EOL>

Indicates whether or not this Extender is capable of retrieving the
specified location.

:param location:
    a URI indicating where to retrieve the TidyPy configuration from
:type location: str
:rtype: bool


@classmethod<EOL><INDENT>def retrieve(cls, location, project_path):<DEDENT>

raise NotImplementedError()<EOL>

Retrieves a TidyPy configuration from the specified location.

:param location:
    a URI indicating where to retrieve the TidyPy configuration from
:type location: str
:param project_path: the full path to the project's base
:type project_path: str
:rtype: dict


@classmethod<EOL><INDENT>def parse(cls, content, is_pyproject=False):<DEDENT>

parsed = pytoml.loads(content)<EOL>if is_pyproject:<EOL><INDENT>parsed = parsed.get('<STR_LIT>', {})<EOL><DEDENT>parsed = parsed.get('<STR_LIT>', {})<EOL>return parsed<EOL>

A convenience method for parsing a TOML-serialized configuration.

:param content: a TOML string containing a TidyPy configuration
:type content: str
:param is_pyproject:
    whether or not the content is (or resembles) a ``pyproject.toml``
    file, where the TidyPy configuration is located within a key named
    ``tool``.
:type is_pyproject: bool
:rtype: dict


def execute_tools(config, path, progress=None):

progress = progress or QuietProgress()<EOL>progress.on_start()<EOL>manager = SyncManager()<EOL>manager.start()<EOL>num_tools = <NUM_LIT:0><EOL>tools = manager.Queue()<EOL>for name, cls in iteritems(get_tools()):<EOL><INDENT>if config[name]['<STR_LIT>'] and cls.can_be_used():<EOL><INDENT>num_tools += <NUM_LIT:1><EOL>tools.put({<EOL>'<STR_LIT:name>': name,<EOL>'<STR_LIT>': config[name],<EOL>})<EOL><DEDENT><DEDENT>collector = Collector(config)<EOL>if not num_tools:<EOL><INDENT>progress.on_finish()<EOL>return collector<EOL><DEDENT>notifications = manager.Queue()<EOL>environment = manager.dict({<EOL>'<STR_LIT>': Finder(path, config),<EOL>})<EOL>workers = []<EOL>for _ in range(config['<STR_LIT>']):<EOL><INDENT>worker = Worker(<EOL>args=(<EOL>tools,<EOL>notifications,<EOL>environment,<EOL>),<EOL>)<EOL>worker.start()<EOL>workers.append(worker)<EOL><DEDENT>while num_tools:<EOL><INDENT>try:<EOL><INDENT>notification = notifications.get(True, <NUM_LIT>)<EOL><DEDENT>except Empty:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>if notification['<STR_LIT:type>'] == '<STR_LIT:start>':<EOL><INDENT>progress.on_tool_start(notification['<STR_LIT>'])<EOL><DEDENT>elif notification['<STR_LIT:type>'] == '<STR_LIT>':<EOL><INDENT>collector.add_issues(notification['<STR_LIT>'])<EOL>progress.on_tool_finish(notification['<STR_LIT>'])<EOL>num_tools -= <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>progress.on_finish()<EOL>return collector<EOL>

Executes the suite of TidyPy tools upon the project and returns the
issues that are found.

:param config: the TidyPy configuration to use
:type config: dict
:param path: that path to the project to analyze
:type path: str
:param progress:
    the progress reporter object that will receive callbacks during the
    execution of the tool suite. If not specified, not progress
    notifications will occur.
:type progress: tidypy.Progress
:rtype: tidypy.Collector


reports = get_reports()<EOL>for report in config.get('<STR_LIT>', []):<EOL><INDENT>if report.get('<STR_LIT:type>') and report['<STR_LIT:type>'] in reports:<EOL><INDENT>cfg = config.get('<STR_LIT>', {}).get(report['<STR_LIT:type>'], {})<EOL>cfg.update(report)<EOL>reporter = reports[report['<STR_LIT:type>']](<EOL>cfg,<EOL>path,<EOL>output_file=output_file,<EOL>)<EOL>reporter.produce(collector)<EOL>if on_report_finish:<EOL><INDENT>on_report_finish(report)<EOL><DEDENT><DEDENT><DEDENT>

Executes the configured suite of issue reports.

:param config: the TidyPy configuration to use
:type config: dict
:param path: that path to the project that was analyzed
:type path: str
:param collector: the issues to report
:type collector: tidypy.Collector