def load_flickr25k_dataset(tag='<STR_LIT>', path="<STR_LIT:data>", n_threads=<NUM_LIT:50>, printable=False):

path = os.path.join(path, '<STR_LIT>')<EOL>filename = '<STR_LIT>'<EOL>url = '<STR_LIT>'<EOL>if folder_exists(os.path.join(path, "<STR_LIT>")) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(path))<EOL>maybe_download_and_extract(filename, path, url, extract=True)<EOL>del_file(os.path.join(path, filename))<EOL><DEDENT>folder_imgs = os.path.join(path, "<STR_LIT>")<EOL>path_imgs = load_file_list(path=folder_imgs, regx='<STR_LIT>', printable=False)<EOL>path_imgs.sort(key=natural_keys)<EOL>folder_tags = os.path.join(path, "<STR_LIT>", "<STR_LIT>", "<STR_LIT>")<EOL>path_tags = load_file_list(path=folder_tags, regx='<STR_LIT>', printable=False)<EOL>path_tags.sort(key=natural_keys)<EOL>if tag is None:<EOL><INDENT>logging.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>logging.info("<STR_LIT>".format(tag))<EOL><DEDENT>images_list = []<EOL>for idx, _v in enumerate(path_tags):<EOL><INDENT>tags = read_file(os.path.join(folder_tags, path_tags[idx])).split('<STR_LIT:\n>')<EOL>if tag is None or tag in tags:<EOL><INDENT>images_list.append(path_imgs[idx])<EOL><DEDENT><DEDENT>images = visualize.read_images(images_list, folder_imgs, n_threads=n_threads, printable=printable)<EOL>return images<EOL>

Load Flickr25K dataset.

    Returns a list of images by a given tag from Flick25k dataset,
    it will download Flickr25k from `the official website <http://press.liacs.nl/mirflickr/mirdownload.html>`__
    at the first time you use it.

    Parameters
    ------------
    tag : str or None
        What images to return.
            - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search <https://www.flickr.com/search/>`__.
            - If you want to get all images, set to ``None``.

    path : str
        The path that the data is downloaded to, defaults is ``data/flickr25k/``.
    n_threads : int
        The number of thread to read image.
    printable : boolean
        Whether to print infomation when reading images, default is ``False``.

    Examples
    -----------
    Get images with tag of sky

    >>> images = tl.files.load_flickr25k_dataset(tag='sky')

    Get all images

    >>> images = tl.files.load_flickr25k_dataset(tag=None, n_threads=100, printable=True)


logging.info("<STR_LIT>".format(path))<EOL>path = os.path.join(path, '<STR_LIT>')<EOL>filename = "<STR_LIT>"<EOL>url = '<STR_LIT>'<EOL>filepath = maybe_download_and_extract(filename, path, url)<EOL>with open(filepath, "<STR_LIT:r>") as f:<EOL><INDENT>words = f.read()<EOL>return words<EOL><DEDENT>

Load Nietzsche dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/nietzsche/``.

    Returns
    --------
    str
        The content.

    Examples
    --------
    >>> see tutorial_generate_text.py
    >>> words = tl.files.load_nietzsche_dataset()
    >>> words = basic_clean_str(words)
    >>> words = words.split()


def load_voc_dataset(path='<STR_LIT:data>', dataset='<STR_LIT>', contain_classes_in_person=False):

path = os.path.join(path, '<STR_LIT>')<EOL>def _recursive_parse_xml_to_dict(xml):<EOL><INDENT>"""<STR_LIT>"""<EOL>if xml is not None:<EOL><INDENT>return {xml.tag: xml.text}<EOL><DEDENT>result = {}<EOL>for child in xml:<EOL><INDENT>child_result = _recursive_parse_xml_to_dict(child)<EOL>if child.tag != '<STR_LIT:object>':<EOL><INDENT>result[child.tag] = child_result[child.tag]<EOL><DEDENT>else:<EOL><INDENT>if child.tag not in result:<EOL><INDENT>result[child.tag] = []<EOL><DEDENT>result[child.tag].append(child_result[child.tag])<EOL><DEDENT><DEDENT>return {xml.tag: result}<EOL><DEDENT>import xml.etree.ElementTree as ET<EOL>if dataset == "<STR_LIT>":<EOL><INDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"  <EOL>logging.info("<STR_LIT>")<EOL><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>extracted_filename = "<STR_LIT>"  <EOL>logging.info("<STR_LIT>")<EOL>logging.info(<EOL>"<STR_LIT>"<EOL>)<EOL>import time<EOL>time.sleep(<NUM_LIT:3>)<EOL>if os.path.isdir(os.path.join(path, extracted_filename)) is False:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>logging.info(<EOL>"<STR_LIT>"<EOL>)<EOL>logging.info("<STR_LIT>" % path)<EOL>exit()<EOL><DEDENT><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if dataset != "<STR_LIT>":<EOL><INDENT>from sys import platform as _platform<EOL>if folder_exists(os.path.join(path, extracted_filename)) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(extracted_filename, path))<EOL>maybe_download_and_extract(tar_filename, path, url, extract=True)<EOL>del_file(os.path.join(path, tar_filename))<EOL>if dataset == "<STR_LIT>":<EOL><INDENT>if _platform == "<STR_LIT:win32>":<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT>else:<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>if _platform == "<STR_LIT:win32>":<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT>else:<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>if _platform == "<STR_LIT:win32>":<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT>else:<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT><DEDENT>del_folder(os.path.join(path, '<STR_LIT>'))<EOL><DEDENT><DEDENT>classes = [<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT:train>", "<STR_LIT>"<EOL>]<EOL>if contain_classes_in_person:<EOL><INDENT>classes_in_person = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>classes_in_person = []<EOL><DEDENT>classes += classes_in_person  <EOL>classes_dict = utils.list_string_to_dict(classes)<EOL>logging.info("<STR_LIT>".format(classes_dict))<EOL>folder_imgs = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_file_list = load_file_list(path=folder_imgs, regx='<STR_LIT>', printable=False)<EOL>logging.info("<STR_LIT>".format(len(imgs_file_list)))<EOL>imgs_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_file_list = [os.path.join(folder_imgs, s) for s in imgs_file_list]<EOL>if dataset != "<STR_LIT>":<EOL><INDENT>folder_semseg = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_semseg_file_list = load_file_list(path=folder_semseg, regx='<STR_LIT>', printable=False)<EOL>logging.info("<STR_LIT>".format(len(imgs_semseg_file_list)))<EOL>imgs_semseg_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_semseg_file_list = [os.path.join(folder_semseg, s) for s in imgs_semseg_file_list]<EOL>folder_insseg = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_insseg_file_list = load_file_list(path=folder_insseg, regx='<STR_LIT>', printable=False)<EOL>logging.info("<STR_LIT>".format(len(imgs_semseg_file_list)))<EOL>imgs_insseg_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_insseg_file_list = [os.path.join(folder_insseg, s) for s in imgs_insseg_file_list]<EOL><DEDENT>else:<EOL><INDENT>imgs_semseg_file_list = []<EOL>imgs_insseg_file_list = []<EOL><DEDENT>folder_ann = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_ann_file_list = load_file_list(path=folder_ann, regx='<STR_LIT>', printable=False)<EOL>logging.info(<EOL>"<STR_LIT>".format(len(imgs_ann_file_list))<EOL>)<EOL>imgs_ann_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_ann_file_list = [os.path.join(folder_ann, s) for s in imgs_ann_file_list]<EOL>if dataset == "<STR_LIT>":  <EOL><INDENT>imgs_file_list_new = []<EOL>for ann in imgs_ann_file_list:<EOL><INDENT>ann = os.path.split(ann)[-<NUM_LIT:1>].split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL>for im in imgs_file_list:<EOL><INDENT>if ann in im:<EOL><INDENT>imgs_file_list_new.append(im)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>imgs_file_list = imgs_file_list_new<EOL>logging.info("<STR_LIT>" % len(imgs_file_list_new))<EOL><DEDENT>def convert(size, box):<EOL><INDENT>dw = <NUM_LIT:1.> / size[<NUM_LIT:0>]<EOL>dh = <NUM_LIT:1.> / size[<NUM_LIT:1>]<EOL>x = (box[<NUM_LIT:0>] + box[<NUM_LIT:1>]) / <NUM_LIT><EOL>y = (box[<NUM_LIT:2>] + box[<NUM_LIT:3>]) / <NUM_LIT><EOL>w = box[<NUM_LIT:1>] - box[<NUM_LIT:0>]<EOL>h = box[<NUM_LIT:3>] - box[<NUM_LIT:2>]<EOL>x = x * dw<EOL>w = w * dw<EOL>y = y * dh<EOL>h = h * dh<EOL>return x, y, w, h<EOL><DEDENT>def convert_annotation(file_name):<EOL><INDENT>"""<STR_LIT>"""<EOL>in_file = open(file_name)<EOL>out_file = "<STR_LIT>"<EOL>tree = ET.parse(in_file)<EOL>root = tree.getroot()<EOL>size = root.find('<STR_LIT:size>')<EOL>w = int(size.find('<STR_LIT:width>').text)<EOL>h = int(size.find('<STR_LIT>').text)<EOL>n_objs = <NUM_LIT:0><EOL>for obj in root.iter('<STR_LIT:object>'):<EOL><INDENT>if dataset != "<STR_LIT>":<EOL><INDENT>difficult = obj.find('<STR_LIT>').text<EOL>cls = obj.find('<STR_LIT:name>').text<EOL>if cls not in classes or int(difficult) == <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>else:<EOL><INDENT>cls = obj.find('<STR_LIT:name>').text<EOL>if cls not in classes:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>cls_id = classes.index(cls)<EOL>xmlbox = obj.find('<STR_LIT>')<EOL>b = (<EOL>float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text),<EOL>float(xmlbox.find('<STR_LIT>').text)<EOL>)<EOL>bb = convert((w, h), b)<EOL>out_file += str(cls_id) + "<STR_LIT:U+0020>" + "<STR_LIT:U+0020>".join([str(a) for a in bb]) + '<STR_LIT:\n>'<EOL>n_objs += <NUM_LIT:1><EOL>if cls in "<STR_LIT>":<EOL><INDENT>for part in obj.iter('<STR_LIT>'):<EOL><INDENT>cls = part.find('<STR_LIT:name>').text<EOL>if cls not in classes_in_person:<EOL><INDENT>continue<EOL><DEDENT>cls_id = classes.index(cls)<EOL>xmlbox = part.find('<STR_LIT>')<EOL>b = (<EOL>float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text),<EOL>float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text)<EOL>)<EOL>bb = convert((w, h), b)<EOL>out_file += str(cls_id) + "<STR_LIT:U+0020>" + "<STR_LIT:U+0020>".join([str(a) for a in bb]) + '<STR_LIT:\n>'<EOL>n_objs += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>in_file.close()<EOL>return n_objs, out_file<EOL><DEDENT>logging.info("<STR_LIT>")<EOL>n_objs_list = []<EOL>objs_info_list = []  <EOL>objs_info_dicts = {}<EOL>for idx, ann_file in enumerate(imgs_ann_file_list):<EOL><INDENT>n_objs, objs_info = convert_annotation(ann_file)<EOL>n_objs_list.append(n_objs)<EOL>objs_info_list.append(objs_info)<EOL>with tf.gfile.GFile(ann_file, '<STR_LIT:r>') as fid:<EOL><INDENT>xml_str = fid.read()<EOL><DEDENT>xml = etree.fromstring(xml_str)<EOL>data = _recursive_parse_xml_to_dict(xml)['<STR_LIT>']<EOL>objs_info_dicts.update({imgs_file_list[idx]: data})<EOL><DEDENT>return imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list, classes, classes_in_person, classes_dict, n_objs_list, objs_info_list, objs_info_dicts<EOL>

Pascal VOC 2007/2012 Dataset.

    It has 20 objects:
    aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow, diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor
    and additional 3 classes : head, hand, foot for person.

    Parameters
    -----------
    path : str
        The path that the data is downloaded to, defaults is ``data/VOC``.
    dataset : str
        The VOC dataset version, `2012`, `2007`, `2007test` or `2012test`. We usually train model on `2007+2012` and test it on `2007test`.
    contain_classes_in_person : boolean
        Whether include head, hand and foot annotation, default is False.

    Returns
    ---------
    imgs_file_list : list of str
        Full paths of all images.
    imgs_semseg_file_list : list of str
        Full paths of all maps for semantic segmentation. Note that not all images have this map!
    imgs_insseg_file_list : list of str
        Full paths of all maps for instance segmentation. Note that not all images have this map!
    imgs_ann_file_list : list of str
        Full paths of all annotations for bounding box and object class, all images have this annotations.
    classes : list of str
        Classes in order.
    classes_in_person : list of str
        Classes in person.
    classes_dict : dictionary
        Class label to integer.
    n_objs_list : list of int
        Number of objects in all images in ``imgs_file_list`` in order.
    objs_info_list : list of str
        Darknet format for the annotation of all images in ``imgs_file_list`` in order. ``[class_id x_centre y_centre width height]`` in ratio format.
    objs_info_dicts : dictionary
        The annotation of all images in ``imgs_file_list``, ``{imgs_file_list : dictionary for annotation}``,
        format from `TensorFlow/Models/object-detection <https://github.com/tensorflow/models/blob/master/object_detection/create_pascal_tf_record.py>`__.

    Examples
    ----------
    >>> imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list,
    >>>     classes, classes_in_person, classes_dict,
    >>>     n_objs_list, objs_info_list, objs_info_dicts = tl.files.load_voc_dataset(dataset="2012", contain_classes_in_person=False)
    >>> idx = 26
    >>> print(classes)
    ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
    >>> print(classes_dict)
    {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
    >>> print(imgs_file_list[idx])
    data/VOC/VOC2012/JPEGImages/2007_000423.jpg
    >>> print(n_objs_list[idx])
    2
    >>> print(imgs_ann_file_list[idx])
    data/VOC/VOC2012/Annotations/2007_000423.xml
    >>> print(objs_info_list[idx])
    14 0.173 0.461333333333 0.142 0.496
    14 0.828 0.542666666667 0.188 0.594666666667
    >>> ann = tl.prepro.parse_darknet_ann_str_to_list(objs_info_list[idx])
    >>> print(ann)
    [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
    >>> c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
    >>> print(c, b)
    [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]

    References
    -------------
    - `Pascal VOC2012 Website <https://pjreddie.com/projects/pascal-voc-dataset-mirror/>`__.
    - `Pascal VOC2007 Website <https://pjreddie.com/projects/pascal-voc-dataset-mirror/>`__.


def _load_mnist_dataset(shape, path, name='<STR_LIT>', url='<STR_LIT>'):

path = os.path.join(path, name)<EOL>def load_mnist_images(path, filename):<EOL><INDENT>filepath = maybe_download_and_extract(filename, path, url)<EOL>logging.info(filepath)<EOL>with gzip.open(filepath, '<STR_LIT:rb>') as f:<EOL><INDENT>data = np.frombuffer(f.read(), np.uint8, offset=<NUM_LIT:16>)<EOL><DEDENT>data = data.reshape(shape)<EOL>return data / np.float32(<NUM_LIT>)<EOL><DEDENT>def load_mnist_labels(path, filename):<EOL><INDENT>filepath = maybe_download_and_extract(filename, path, url)<EOL>with gzip.open(filepath, '<STR_LIT:rb>') as f:<EOL><INDENT>data = np.frombuffer(f.read(), np.uint8, offset=<NUM_LIT:8>)<EOL><DEDENT>return data<EOL><DEDENT>logging.info("<STR_LIT>".format(name.upper(), path))<EOL>X_train = load_mnist_images(path, '<STR_LIT>')<EOL>y_train = load_mnist_labels(path, '<STR_LIT>')<EOL>X_test = load_mnist_images(path, '<STR_LIT>')<EOL>y_test = load_mnist_labels(path, '<STR_LIT>')<EOL>X_train, X_val = X_train[:-<NUM_LIT>], X_train[-<NUM_LIT>:]<EOL>y_train, y_val = y_train[:-<NUM_LIT>], y_train[-<NUM_LIT>:]<EOL>X_train = np.asarray(X_train, dtype=np.float32)<EOL>y_train = np.asarray(y_train, dtype=np.int32)<EOL>X_val = np.asarray(X_val, dtype=np.float32)<EOL>y_val = np.asarray(y_val, dtype=np.int32)<EOL>X_test = np.asarray(X_test, dtype=np.float32)<EOL>y_test = np.asarray(y_test, dtype=np.int32)<EOL>return X_train, y_train, X_val, y_val, X_test, y_test<EOL>

A generic function to load mnist-like dataset.

    Parameters:
    ----------
    shape : tuple
        The shape of digit images.
    path : str
        The path that the data is downloaded to.
    name : str
        The dataset name you want to use(the default is 'mnist').
    url : str
        The url of dataset(the default is 'http://yann.lecun.com/exdb/mnist/').


def load_mnist_dataset(shape=(-<NUM_LIT:1>, <NUM_LIT>), path='<STR_LIT:data>'):

return _load_mnist_dataset(shape, path, name='<STR_LIT>', url='<STR_LIT>')<EOL>

Load the original mnist.

    Automatically download MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 digit images respectively.

    Parameters
    ----------
    shape : tuple
        The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).
    path : str
        The path that the data is downloaded to.

    Returns
    -------
    X_train, y_train, X_val, y_val, X_test, y_test: tuple
        Return splitted training/validation/test set respectively.

    Examples
    --------
    >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1,784), path='datasets')
    >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_mnist_dataset(shape=(-1, 28, 28, 1))


def load_fashion_mnist_dataset(shape=(-<NUM_LIT:1>, <NUM_LIT>), path='<STR_LIT:data>'):

return _load_mnist_dataset(<EOL>shape, path, name='<STR_LIT>', url='<STR_LIT>'<EOL>)<EOL>

Load the fashion mnist.

    Automatically download fashion-MNIST dataset and return the training, validation and test set with 50000, 10000 and 10000 fashion images respectively, `examples <http://marubon-ds.blogspot.co.uk/2017/09/fashion-mnist-exploring.html>`__.

    Parameters
    ----------
    shape : tuple
        The shape of digit images (the default is (-1, 784), alternatively (-1, 28, 28, 1)).
    path : str
        The path that the data is downloaded to.

    Returns
    -------
    X_train, y_train, X_val, y_val, X_test, y_test: tuple
        Return splitted training/validation/test set respectively.

    Examples
    --------
    >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1,784), path='datasets')
    >>> X_train, y_train, X_val, y_val, X_test, y_test = tl.files.load_fashion_mnist_dataset(shape=(-1, 28, 28, 1))


def load_cifar10_dataset(shape=(-<NUM_LIT:1>, <NUM_LIT:32>, <NUM_LIT:32>, <NUM_LIT:3>), path='<STR_LIT:data>', plotable=False):

path = os.path.join(path, '<STR_LIT>')<EOL>logging.info("<STR_LIT>".format(path))<EOL>def unpickle(file):<EOL><INDENT>fp = open(file, '<STR_LIT:rb>')<EOL>if sys.version_info.major == <NUM_LIT:2>:<EOL><INDENT>data = pickle.load(fp)<EOL><DEDENT>elif sys.version_info.major == <NUM_LIT:3>:<EOL><INDENT>data = pickle.load(fp, encoding='<STR_LIT>')<EOL><DEDENT>fp.close()<EOL>return data<EOL><DEDENT>filename = '<STR_LIT>'<EOL>url = '<STR_LIT>'<EOL>maybe_download_and_extract(filename, path, url, extract=True)<EOL>X_train = None<EOL>y_train = []<EOL>for i in range(<NUM_LIT:1>, <NUM_LIT:6>):<EOL><INDENT>data_dic = unpickle(os.path.join(path, '<STR_LIT>', "<STR_LIT>".format(i)))<EOL>if i == <NUM_LIT:1>:<EOL><INDENT>X_train = data_dic['<STR_LIT:data>']<EOL><DEDENT>else:<EOL><INDENT>X_train = np.vstack((X_train, data_dic['<STR_LIT:data>']))<EOL><DEDENT>y_train += data_dic['<STR_LIT>']<EOL><DEDENT>test_data_dic = unpickle(os.path.join(path, '<STR_LIT>', "<STR_LIT>"))<EOL>X_test = test_data_dic['<STR_LIT:data>']<EOL>y_test = np.array(test_data_dic['<STR_LIT>'])<EOL>if shape == (-<NUM_LIT:1>, <NUM_LIT:3>, <NUM_LIT:32>, <NUM_LIT:32>):<EOL><INDENT>X_test = X_test.reshape(shape)<EOL>X_train = X_train.reshape(shape)<EOL><DEDENT>elif shape == (-<NUM_LIT:1>, <NUM_LIT:32>, <NUM_LIT:32>, <NUM_LIT:3>):<EOL><INDENT>X_test = X_test.reshape(shape, order='<STR_LIT:F>')<EOL>X_train = X_train.reshape(shape, order='<STR_LIT:F>')<EOL>X_test = np.transpose(X_test, (<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>, <NUM_LIT:3>))<EOL>X_train = np.transpose(X_train, (<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>, <NUM_LIT:3>))<EOL><DEDENT>else:<EOL><INDENT>X_test = X_test.reshape(shape)<EOL>X_train = X_train.reshape(shape)<EOL><DEDENT>y_train = np.array(y_train)<EOL>if plotable:<EOL><INDENT>logging.info('<STR_LIT>')<EOL>fig = plt.figure(<NUM_LIT:1>)<EOL>logging.info('<STR_LIT>' % X_train[<NUM_LIT:0>].shape)<EOL>plt.ion()  <EOL>count = <NUM_LIT:1><EOL>for _ in range(<NUM_LIT:10>):  <EOL><INDENT>for _ in range(<NUM_LIT:10>):  <EOL><INDENT>_ = fig.add_subplot(<NUM_LIT:10>, <NUM_LIT:10>, count)<EOL>if shape == (-<NUM_LIT:1>, <NUM_LIT:3>, <NUM_LIT:32>, <NUM_LIT:32>):<EOL><INDENT>plt.imshow(np.transpose(X_train[count - <NUM_LIT:1>], (<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:0>)), interpolation='<STR_LIT>')<EOL><DEDENT>elif shape == (-<NUM_LIT:1>, <NUM_LIT:32>, <NUM_LIT:32>, <NUM_LIT:3>):<EOL><INDENT>plt.imshow(X_train[count - <NUM_LIT:1>], interpolation='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>plt.gca().xaxis.set_major_locator(plt.NullLocator())  <EOL>plt.gca().yaxis.set_major_locator(plt.NullLocator())<EOL>count = count + <NUM_LIT:1><EOL><DEDENT><DEDENT>plt.draw()  <EOL>plt.pause(<NUM_LIT:3>)  <EOL>logging.info("<STR_LIT>" % X_train.shape)<EOL>logging.info("<STR_LIT>" % y_train.shape)<EOL>logging.info("<STR_LIT>" % X_test.shape)<EOL>logging.info("<STR_LIT>" % y_test.shape)<EOL><DEDENT>X_train = np.asarray(X_train, dtype=np.float32)<EOL>X_test = np.asarray(X_test, dtype=np.float32)<EOL>y_train = np.asarray(y_train, dtype=np.int32)<EOL>y_test = np.asarray(y_test, dtype=np.int32)<EOL>return X_train, y_train, X_test, y_test<EOL>

Load CIFAR-10 dataset.

    It consists of 60000 32x32 colour images in 10 classes, with
    6000 images per class. There are 50000 training images and 10000 test images.

    The dataset is divided into five training batches and one test batch, each with
    10000 images. The test batch contains exactly 1000 randomly-selected images from
    each class. The training batches contain the remaining images in random order,
    but some training batches may contain more images from one class than another.
    Between them, the training batches contain exactly 5000 images from each class.

    Parameters
    ----------
    shape : tupe
        The shape of digit images e.g. (-1, 3, 32, 32) and (-1, 32, 32, 3).
    path : str
        The path that the data is downloaded to, defaults is ``data/cifar10/``.
    plotable : boolean
        Whether to plot some image examples, False as default.

    Examples
    --------
    >>> X_train, y_train, X_test, y_test = tl.files.load_cifar10_dataset(shape=(-1, 32, 32, 3))

    References
    ----------
    - `CIFAR website <https://www.cs.toronto.edu/~kriz/cifar.html>`__
    - `Data download link <https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz>`__
    - `<https://teratail.com/questions/28932>`__


start_time = time.time()<EOL>path = os.path.join(path, '<STR_LIT>')<EOL>logging.info("<STR_LIT>".format(path, include_extra))<EOL>url = "<STR_LIT>"<EOL>np_file = os.path.join(path, "<STR_LIT>")<EOL>if file_exists(np_file) is False:<EOL><INDENT>filename = "<STR_LIT>"<EOL>filepath = maybe_download_and_extract(filename, path, url)<EOL>mat = sio.loadmat(filepath)<EOL>X_train = mat['<STR_LIT:X>'] / <NUM_LIT>  <EOL>X_train = np.transpose(X_train, (<NUM_LIT:3>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:2>))<EOL>y_train = np.squeeze(mat['<STR_LIT:y>'], axis=<NUM_LIT:1>)<EOL>y_train[y_train == <NUM_LIT:10>] = <NUM_LIT:0>  <EOL>np.savez(np_file, X=X_train, y=y_train)<EOL>del_file(filepath)<EOL><DEDENT>else:<EOL><INDENT>v = np.load(np_file)<EOL>X_train = v['<STR_LIT:X>']<EOL>y_train = v['<STR_LIT:y>']<EOL><DEDENT>logging.info("<STR_LIT>".format(len(y_train)))<EOL>np_file = os.path.join(path, "<STR_LIT>")<EOL>if file_exists(np_file) is False:<EOL><INDENT>filename = "<STR_LIT>"<EOL>filepath = maybe_download_and_extract(filename, path, url)<EOL>mat = sio.loadmat(filepath)<EOL>X_test = mat['<STR_LIT:X>'] / <NUM_LIT><EOL>X_test = np.transpose(X_test, (<NUM_LIT:3>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:2>))<EOL>y_test = np.squeeze(mat['<STR_LIT:y>'], axis=<NUM_LIT:1>)<EOL>y_test[y_test == <NUM_LIT:10>] = <NUM_LIT:0><EOL>np.savez(np_file, X=X_test, y=y_test)<EOL>del_file(filepath)<EOL><DEDENT>else:<EOL><INDENT>v = np.load(np_file)<EOL>X_test = v['<STR_LIT:X>']<EOL>y_test = v['<STR_LIT:y>']<EOL><DEDENT>logging.info("<STR_LIT>".format(len(y_test)))<EOL>if include_extra:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>np_file = os.path.join(path, "<STR_LIT>")<EOL>if file_exists(np_file) is False:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>filename = "<STR_LIT>"<EOL>filepath = maybe_download_and_extract(filename, path, url)<EOL>mat = sio.loadmat(filepath)<EOL>X_extra = mat['<STR_LIT:X>'] / <NUM_LIT><EOL>X_extra = np.transpose(X_extra, (<NUM_LIT:3>, <NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:2>))<EOL>y_extra = np.squeeze(mat['<STR_LIT:y>'], axis=<NUM_LIT:1>)<EOL>y_extra[y_extra == <NUM_LIT:10>] = <NUM_LIT:0><EOL>np.savez(np_file, X=X_extra, y=y_extra)<EOL>del_file(filepath)<EOL><DEDENT>else:<EOL><INDENT>v = np.load(np_file)<EOL>X_extra = v['<STR_LIT:X>']<EOL>y_extra = v['<STR_LIT:y>']<EOL><DEDENT>logging.info("<STR_LIT>".format(len(y_extra), len(y_train)))<EOL>t = time.time()<EOL>X_train = np.concatenate((X_train, X_extra), <NUM_LIT:0>)<EOL>y_train = np.concatenate((y_train, y_extra), <NUM_LIT:0>)<EOL>logging.info("<STR_LIT>".format(len(y_extra), len(y_train), time.time() - t))<EOL><DEDENT>else:<EOL><INDENT>logging.info("<STR_LIT>")<EOL><DEDENT>logging.info("<STR_LIT>" % (str(X_train.shape[<NUM_LIT:1>:<NUM_LIT:4>]), len(y_train), len(y_test)))<EOL>logging.info("<STR_LIT>".format(int(time.time() - start_time)))<EOL>return X_train, y_train, X_test, y_test<EOL>

Load Cropped SVHN.

    The Cropped Street View House Numbers (SVHN) Dataset contains 32x32x3 RGB images.
    Digit '1' has label 1, '9' has label 9 and '0' has label 0 (the original dataset uses 10 to represent '0'), see `ufldl website <http://ufldl.stanford.edu/housenumbers/>`__.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to.
    include_extra : boolean
        If True (default), add extra images to the training set.

    Returns
    -------
    X_train, y_train, X_test, y_test: tuple
        Return splitted training/test set respectively.

    Examples
    ---------
    >>> X_train, y_train, X_test, y_test = tl.files.load_cropped_svhn(include_extra=False)
    >>> tl.vis.save_images(X_train[0:100], [10, 10], 'svhn.png')


path = os.path.join(path, '<STR_LIT>')<EOL>logging.info("<STR_LIT>".format(path))<EOL>filename = '<STR_LIT>'<EOL>url = '<STR_LIT>'<EOL>maybe_download_and_extract(filename, path, url, extract=True)<EOL>data_path = os.path.join(path, '<STR_LIT>', '<STR_LIT:data>')<EOL>train_path = os.path.join(data_path, "<STR_LIT>")<EOL>valid_path = os.path.join(data_path, "<STR_LIT>")<EOL>test_path = os.path.join(data_path, "<STR_LIT>")<EOL>word_to_id = nlp.build_vocab(nlp.read_words(train_path))<EOL>train_data = nlp.words_to_word_ids(nlp.read_words(train_path), word_to_id)<EOL>valid_data = nlp.words_to_word_ids(nlp.read_words(valid_path), word_to_id)<EOL>test_data = nlp.words_to_word_ids(nlp.read_words(test_path), word_to_id)<EOL>vocab_size = len(word_to_id)<EOL>return train_data, valid_data, test_data, vocab_size<EOL>

Load Penn TreeBank (PTB) dataset.

    It is used in many LANGUAGE MODELING papers,
    including "Empirical Evaluation and Combination of Advanced Language
    Modeling Techniques", "Recurrent Neural Network Regularization".
    It consists of 929k training words, 73k validation words, and 82k test
    words. It has 10k words in its vocabulary.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/ptb/``.

    Returns
    --------
    train_data, valid_data, test_data : list of int
        The training, validating and testing data in integer format.
    vocab_size : int
        The vocabulary size.

    Examples
    --------
    >>> train_data, valid_data, test_data, vocab_size = tl.files.load_ptb_dataset()

    References
    ---------------
    - ``tensorflow.models.rnn.ptb import reader``
    - `Manual download <http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz>`__

    Notes
    ------
    - If you want to get the raw data, see the source code.


path = os.path.join(path, '<STR_LIT>')<EOL>logging.info("<STR_LIT>".format(path))<EOL>filename = '<STR_LIT>'<EOL>url = '<STR_LIT>'<EOL>maybe_download_and_extract(filename, path, url, expected_bytes=<NUM_LIT>)<EOL>with zipfile.ZipFile(os.path.join(path, filename)) as f:<EOL><INDENT>word_list = f.read(f.namelist()[<NUM_LIT:0>]).split()<EOL>for idx, _ in enumerate(word_list):<EOL><INDENT>word_list[idx] = word_list[idx].decode()<EOL><DEDENT><DEDENT>return word_list<EOL>

Load Matt Mahoney's dataset.

    Download a text file from Matt Mahoney's website
    if not present, and make sure it's the right size.
    Extract the first file enclosed in a zip file as a list of words.
    This dataset can be used for Word Embedding.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/mm_test8/``.

    Returns
    --------
    list of str
        The raw text data e.g. [.... 'their', 'families', 'who', 'were', 'expelled', 'from', 'jerusalem', ...]

    Examples
    --------
    >>> words = tl.files.load_matt_mahoney_text8_dataset()
    >>> print('Data size', len(words))


path = os.path.join(path, '<STR_LIT>')<EOL>_WMT_ENFR_TRAIN_URL = "<STR_LIT>"<EOL>_WMT_ENFR_DEV_URL = "<STR_LIT>"<EOL>def gunzip_file(gz_path, new_path):<EOL><INDENT>"""<STR_LIT>"""<EOL>logging.info("<STR_LIT>" % (gz_path, new_path))<EOL>with gzip.open(gz_path, "<STR_LIT:rb>") as gz_file:<EOL><INDENT>with open(new_path, "<STR_LIT:wb>") as new_file:<EOL><INDENT>for line in gz_file:<EOL><INDENT>new_file.write(line)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>def get_wmt_enfr_train_set(path):<EOL><INDENT>"""<STR_LIT>"""<EOL>filename = "<STR_LIT>"<EOL>maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)<EOL>train_path = os.path.join(path, "<STR_LIT>")<EOL>gunzip_file(train_path + "<STR_LIT>", train_path + "<STR_LIT>")<EOL>gunzip_file(train_path + "<STR_LIT>", train_path + "<STR_LIT>")<EOL>return train_path<EOL><DEDENT>def get_wmt_enfr_dev_set(path):<EOL><INDENT>"""<STR_LIT>"""<EOL>filename = "<STR_LIT>"<EOL>dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)<EOL>dev_name = "<STR_LIT>"<EOL>dev_path = os.path.join(path, "<STR_LIT>")<EOL>if not (gfile.Exists(dev_path + "<STR_LIT>") and gfile.Exists(dev_path + "<STR_LIT>")):<EOL><INDENT>logging.info("<STR_LIT>" % dev_file)<EOL>with tarfile.open(dev_file, "<STR_LIT>") as dev_tar:<EOL><INDENT>fr_dev_file = dev_tar.getmember("<STR_LIT>" + dev_name + "<STR_LIT>")<EOL>en_dev_file = dev_tar.getmember("<STR_LIT>" + dev_name + "<STR_LIT>")<EOL>fr_dev_file.name = dev_name + "<STR_LIT>"  <EOL>en_dev_file.name = dev_name + "<STR_LIT>"<EOL>dev_tar.extract(fr_dev_file, path)<EOL>dev_tar.extract(en_dev_file, path)<EOL><DEDENT><DEDENT>return dev_path<EOL><DEDENT>logging.info("<STR_LIT>".format(path))<EOL>train_path = get_wmt_enfr_train_set(path)<EOL>dev_path = get_wmt_enfr_dev_set(path)<EOL>return train_path, dev_path<EOL>

Load WMT'15 English-to-French translation dataset.

    It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.
    Returns the directories of training data and test data.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.

    References
    ----------
    - Code modified from /tensorflow/models/rnn/translation/data_utils.py

    Notes
    -----
    Usually, it will take a long time to download this dataset.


def load_flickr1M_dataset(tag='<STR_LIT>', size=<NUM_LIT:10>, path="<STR_LIT:data>", n_threads=<NUM_LIT:50>, printable=False):

path = os.path.join(path, '<STR_LIT>')<EOL>logging.info("<STR_LIT>".format(size * <NUM_LIT:10>, size * <NUM_LIT>))<EOL>images_zip = [<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>',<EOL>'<STR_LIT>', '<STR_LIT>', '<STR_LIT>'<EOL>]<EOL>tag_zip = '<STR_LIT>'<EOL>url = '<STR_LIT>'<EOL>for image_zip in images_zip[<NUM_LIT:0>:size]:<EOL><INDENT>image_folder = image_zip.split("<STR_LIT:.>")[<NUM_LIT:0>]<EOL>if folder_exists(os.path.join(path, image_folder)) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(image_folder, path))<EOL>maybe_download_and_extract(image_zip, path, url, extract=True)<EOL>del_file(os.path.join(path, image_zip))<EOL>shutil.move(os.path.join(path, '<STR_LIT>'), os.path.join(path, image_folder))<EOL><DEDENT>else:<EOL><INDENT>logging.info("<STR_LIT>".format(image_folder, path))<EOL><DEDENT><DEDENT>if folder_exists(os.path.join(path, "<STR_LIT>")) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(path))<EOL>maybe_download_and_extract(tag_zip, path, url, extract=True)<EOL>del_file(os.path.join(path, tag_zip))<EOL><DEDENT>else:<EOL><INDENT>logging.info("<STR_LIT>".format(path))<EOL><DEDENT>images_list = []<EOL>images_folder_list = []<EOL>for i in range(<NUM_LIT:0>, size):<EOL><INDENT>images_folder_list += load_folder_list(path=os.path.join(path, '<STR_LIT>' % i))<EOL><DEDENT>images_folder_list.sort(key=lambda s: int(s.split('<STR_LIT:/>')[-<NUM_LIT:1>]))  <EOL>for folder in images_folder_list[<NUM_LIT:0>:size * <NUM_LIT:10>]:<EOL><INDENT>tmp = load_file_list(path=folder, regx='<STR_LIT>', printable=False)<EOL>tmp.sort(key=lambda s: int(s.split('<STR_LIT:.>')[-<NUM_LIT:2>]))  <EOL>images_list.extend([os.path.join(folder, x) for x in tmp])<EOL><DEDENT>tag_list = []<EOL>tag_folder_list = load_folder_list(os.path.join(path, "<STR_LIT>"))<EOL>tag_folder_list.sort(key=lambda s: int(os.path.basename(s)))<EOL>for folder in tag_folder_list[<NUM_LIT:0>:size * <NUM_LIT:10>]:<EOL><INDENT>tmp = load_file_list(path=folder, regx='<STR_LIT>', printable=False)<EOL>tmp.sort(key=lambda s: int(s.split('<STR_LIT:.>')[-<NUM_LIT:2>]))  <EOL>tmp = [os.path.join(folder, s) for s in tmp]<EOL>tag_list += tmp<EOL><DEDENT>logging.info("<STR_LIT>".format(tag))<EOL>select_images_list = []<EOL>for idx, _val in enumerate(tag_list):<EOL><INDENT>tags = read_file(tag_list[idx]).split('<STR_LIT:\n>')<EOL>if tag in tags:<EOL><INDENT>select_images_list.append(images_list[idx])<EOL><DEDENT><DEDENT>logging.info("<STR_LIT>".format(tag))<EOL>images = visualize.read_images(select_images_list, '<STR_LIT>', n_threads=n_threads, printable=printable)<EOL>return images<EOL>

Load Flick1M dataset.

    Returns a list of images by a given tag from Flickr1M dataset,
    it will download Flickr1M from `the official website <http://press.liacs.nl/mirflickr/mirdownload.html>`__
    at the first time you use it.

    Parameters
    ------------
    tag : str or None
        What images to return.
            - If you want to get images with tag, use string like 'dog', 'red', see `Flickr Search <https://www.flickr.com/search/>`__.
            - If you want to get all images, set to ``None``.

    size : int
        integer between 1 to 10. 1 means 100k images ... 5 means 500k images, 10 means all 1 million images. Default is 10.
    path : str
        The path that the data is downloaded to, defaults is ``data/flickr25k/``.
    n_threads : int
        The number of thread to read image.
    printable : boolean
        Whether to print infomation when reading images, default is ``False``.

    Examples
    ----------
    Use 200k images

    >>> images = tl.files.load_flickr1M_dataset(tag='zebra', size=2)

    Use 1 Million images

    >>> images = tl.files.load_flickr1M_dataset(tag='zebra')


path = os.path.join(path, '<STR_LIT>')<EOL>url = '<STR_LIT>'<EOL>if folder_exists(os.path.join(path, filename)) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(filename, path))<EOL>maybe_download_and_extract(filename + '<STR_LIT>', path, url, extract=True)<EOL>del_file(os.path.join(path, filename + '<STR_LIT>'))<EOL><DEDENT>def load_image_from_folder(path):<EOL><INDENT>path_imgs = load_file_list(path=path, regx='<STR_LIT>', printable=False)<EOL>return visualize.read_images(path_imgs, path=path, n_threads=<NUM_LIT:10>, printable=False)<EOL><DEDENT>im_train_A = load_image_from_folder(os.path.join(path, filename, "<STR_LIT>"))<EOL>im_train_B = load_image_from_folder(os.path.join(path, filename, "<STR_LIT>"))<EOL>im_test_A = load_image_from_folder(os.path.join(path, filename, "<STR_LIT>"))<EOL>im_test_B = load_image_from_folder(os.path.join(path, filename, "<STR_LIT>"))<EOL>def if_2d_to_3d(images):  <EOL><INDENT>for i, _v in enumerate(images):<EOL><INDENT>if len(images[i].shape) == <NUM_LIT:2>:<EOL><INDENT>images[i] = images[i][:, :, np.newaxis]<EOL>images[i] = np.tile(images[i], (<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:3>))<EOL><DEDENT><DEDENT>return images<EOL><DEDENT>im_train_A = if_2d_to_3d(im_train_A)<EOL>im_train_B = if_2d_to_3d(im_train_B)<EOL>im_test_A = if_2d_to_3d(im_test_A)<EOL>im_test_B = if_2d_to_3d(im_test_B)<EOL>return im_train_A, im_train_B, im_test_A, im_test_B<EOL>

Load images from CycleGAN's database, see `this link <https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/>`__.

    Parameters
    ------------
    filename : str
        The dataset you want, see `this link <https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets/>`__.
    path : str
        The path that the data is downloaded to, defaults is `data/cyclegan`

    Examples
    ---------
    >>> im_train_A, im_train_B, im_test_A, im_test_B = load_cyclegan_dataset(filename='summer2winter_yosemite')


def save_response_content(response, destination, chunk_size=<NUM_LIT:32> * <NUM_LIT>):<EOL><INDENT>total_size = int(response.headers.get('<STR_LIT>', <NUM_LIT:0>))<EOL>with open(destination, "<STR_LIT:wb>") as f:<EOL><INDENT>for chunk in tqdm(response.iter_content(chunk_size), total=total_size, unit='<STR_LIT:B>', unit_scale=True,<EOL>desc=destination):<EOL><INDENT>if chunk:  <EOL><INDENT>f.write(chunk)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>def get_confirm_token(response):<EOL><INDENT>for key, value in response.cookies.items():<EOL><INDENT>if key.startswith('<STR_LIT>'):<EOL><INDENT>return value<EOL><DEDENT><DEDENT>return None<EOL><DEDENT>URL = "<STR_LIT>"<EOL>session = requests.Session()<EOL>response = session.get(URL, params={'<STR_LIT:id>': ID}, stream=True)<EOL>token = get_confirm_token(response)<EOL>if token:<EOL><INDENT>params = {'<STR_LIT:id>': ID, '<STR_LIT>': token}<EOL>response = session.get(URL, params=params, stream=True)<EOL><DEDENT>save_response_content(response, destination)<EOL>

Download file from Google Drive.

    See ``tl.files.load_celebA_dataset`` for example.

    Parameters
    --------------
    ID : str
        The driver ID.
    destination : str
        The destination for save file.


data_dir = '<STR_LIT>'<EOL>filename, drive_id = "<STR_LIT>", "<STR_LIT>"<EOL>save_path = os.path.join(path, filename)<EOL>image_path = os.path.join(path, data_dir)<EOL>if os.path.exists(image_path):<EOL><INDENT>logging.info('<STR_LIT>'.format(save_path))<EOL><DEDENT>else:<EOL><INDENT>exists_or_mkdir(path)<EOL>download_file_from_google_drive(drive_id, save_path)<EOL>zip_dir = '<STR_LIT>'<EOL>with zipfile.ZipFile(save_path) as zf:<EOL><INDENT>zip_dir = zf.namelist()[<NUM_LIT:0>]<EOL>zf.extractall(path)<EOL><DEDENT>os.remove(save_path)<EOL>os.rename(os.path.join(path, zip_dir), image_path)<EOL><DEDENT>data_files = load_file_list(path=image_path, regx='<STR_LIT>', printable=False)<EOL>for i, _v in enumerate(data_files):<EOL><INDENT>data_files[i] = os.path.join(image_path, data_files[i])<EOL><DEDENT>return data_files<EOL>

Load CelebA dataset

    Return a list of image path.

    Parameters
    -----------
    path : str
        The path that the data is downloaded to, defaults is ``data/celebA/``.


path = os.path.join(path, '<STR_LIT>')<EOL>def _recursive_parse_xml_to_dict(xml):<EOL><INDENT>"""<STR_LIT>"""<EOL>if not xml:<EOL><INDENT>return {xml.tag: xml.text}<EOL><DEDENT>result = {}<EOL>for child in xml:<EOL><INDENT>child_result = _recursive_parse_xml_to_dict(child)<EOL>if child.tag != '<STR_LIT:object>':<EOL><INDENT>result[child.tag] = child_result[child.tag]<EOL><DEDENT>else:<EOL><INDENT>if child.tag not in result:<EOL><INDENT>result[child.tag] = []<EOL><DEDENT>result[child.tag].append(child_result[child.tag])<EOL><DEDENT><DEDENT>return {xml.tag: result}<EOL><DEDENT>if dataset == "<STR_LIT>":<EOL><INDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"  <EOL>logging.info("<STR_LIT>")<EOL><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>extracted_filename = "<STR_LIT>"  <EOL>logging.info("<STR_LIT>")<EOL>logging.info(<EOL>"<STR_LIT>"<EOL>)<EOL>time.sleep(<NUM_LIT:3>)<EOL>if os.path.isdir(os.path.join(path, extracted_filename)) is False:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>logging.info(<EOL>"<STR_LIT>"<EOL>)<EOL>logging.info("<STR_LIT>" % path)<EOL>exit()<EOL><DEDENT><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if dataset != "<STR_LIT>":<EOL><INDENT>_platform = sys.platform<EOL>if folder_exists(os.path.join(path, extracted_filename)) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(extracted_filename, path))<EOL>maybe_download_and_extract(tar_filename, path, url, extract=True)<EOL>del_file(os.path.join(path, tar_filename))<EOL>if dataset == "<STR_LIT>":<EOL><INDENT>if _platform == "<STR_LIT:win32>":<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT>else:<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>if _platform == "<STR_LIT:win32>":<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT>else:<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT><DEDENT>elif dataset == "<STR_LIT>":<EOL><INDENT>if _platform == "<STR_LIT:win32>":<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT>else:<EOL><INDENT>os.system("<STR_LIT>".format(path, path))<EOL><DEDENT><DEDENT>del_folder(os.path.join(path, '<STR_LIT>'))<EOL><DEDENT><DEDENT>classes = [<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>",<EOL>"<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT:train>", "<STR_LIT>"<EOL>]<EOL>if contain_classes_in_person:<EOL><INDENT>classes_in_person = ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>classes_in_person = []<EOL><DEDENT>classes += classes_in_person  <EOL>classes_dict = utils.list_string_to_dict(classes)<EOL>logging.info("<STR_LIT>".format(classes_dict))<EOL>folder_imgs = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_file_list = load_file_list(path=folder_imgs, regx='<STR_LIT>', printable=False)<EOL>logging.info("<STR_LIT>".format(len(imgs_file_list)))<EOL>imgs_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_file_list = [os.path.join(folder_imgs, s) for s in imgs_file_list]<EOL>if dataset != "<STR_LIT>":<EOL><INDENT>folder_semseg = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_semseg_file_list = load_file_list(path=folder_semseg, regx='<STR_LIT>', printable=False)<EOL>logging.info("<STR_LIT>".format(len(imgs_semseg_file_list)))<EOL>imgs_semseg_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_semseg_file_list = [os.path.join(folder_semseg, s) for s in imgs_semseg_file_list]<EOL>folder_insseg = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_insseg_file_list = load_file_list(path=folder_insseg, regx='<STR_LIT>', printable=False)<EOL>logging.info("<STR_LIT>".format(len(imgs_semseg_file_list)))<EOL>imgs_insseg_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_insseg_file_list = [os.path.join(folder_insseg, s) for s in imgs_insseg_file_list]<EOL><DEDENT>else:<EOL><INDENT>imgs_semseg_file_list = []<EOL>imgs_insseg_file_list = []<EOL><DEDENT>folder_ann = os.path.join(path, extracted_filename, "<STR_LIT>")<EOL>imgs_ann_file_list = load_file_list(path=folder_ann, regx='<STR_LIT>', printable=False)<EOL>logging.info(<EOL>"<STR_LIT>".format(len(imgs_ann_file_list))<EOL>)<EOL>imgs_ann_file_list.sort(<EOL>key=lambda s: int(s.replace('<STR_LIT:.>', '<STR_LIT:U+0020>').replace('<STR_LIT:_>', '<STR_LIT>').split('<STR_LIT:U+0020>')[-<NUM_LIT:2>])<EOL>)  <EOL>imgs_ann_file_list = [os.path.join(folder_ann, s) for s in imgs_ann_file_list]<EOL>if dataset == "<STR_LIT>":  <EOL><INDENT>imgs_file_list_new = []<EOL>for ann in imgs_ann_file_list:<EOL><INDENT>ann = os.path.split(ann)[-<NUM_LIT:1>].split('<STR_LIT:.>')[<NUM_LIT:0>]<EOL>for im in imgs_file_list:<EOL><INDENT>if ann in im:<EOL><INDENT>imgs_file_list_new.append(im)<EOL>break<EOL><DEDENT><DEDENT><DEDENT>imgs_file_list = imgs_file_list_new<EOL>logging.info("<STR_LIT>" % len(imgs_file_list_new))<EOL><DEDENT>def convert(size, box):<EOL><INDENT>dw = <NUM_LIT:1.> / size[<NUM_LIT:0>]<EOL>dh = <NUM_LIT:1.> / size[<NUM_LIT:1>]<EOL>x = (box[<NUM_LIT:0>] + box[<NUM_LIT:1>]) / <NUM_LIT><EOL>y = (box[<NUM_LIT:2>] + box[<NUM_LIT:3>]) / <NUM_LIT><EOL>w = box[<NUM_LIT:1>] - box[<NUM_LIT:0>]<EOL>h = box[<NUM_LIT:3>] - box[<NUM_LIT:2>]<EOL>x = x * dw<EOL>w = w * dw<EOL>y = y * dh<EOL>h = h * dh<EOL>return x, y, w, h<EOL><DEDENT>def convert_annotation(file_name):<EOL><INDENT>"""<STR_LIT>"""<EOL>in_file = open(file_name)<EOL>out_file = "<STR_LIT>"<EOL>tree = ET.parse(in_file)<EOL>root = tree.getroot()<EOL>size = root.find('<STR_LIT:size>')<EOL>w = int(size.find('<STR_LIT:width>').text)<EOL>h = int(size.find('<STR_LIT>').text)<EOL>n_objs = <NUM_LIT:0><EOL>for obj in root.iter('<STR_LIT:object>'):<EOL><INDENT>if dataset != "<STR_LIT>":<EOL><INDENT>difficult = obj.find('<STR_LIT>').text<EOL>cls = obj.find('<STR_LIT:name>').text<EOL>if cls not in classes or int(difficult) == <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>else:<EOL><INDENT>cls = obj.find('<STR_LIT:name>').text<EOL>if cls not in classes:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>cls_id = classes.index(cls)<EOL>xmlbox = obj.find('<STR_LIT>')<EOL>b = (<EOL>float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text),<EOL>float(xmlbox.find('<STR_LIT>').text)<EOL>)<EOL>bb = convert((w, h), b)<EOL>out_file += str(cls_id) + "<STR_LIT:U+0020>" + "<STR_LIT:U+0020>".join([str(a) for a in bb]) + '<STR_LIT:\n>'<EOL>n_objs += <NUM_LIT:1><EOL>if cls in "<STR_LIT>":<EOL><INDENT>for part in obj.iter('<STR_LIT>'):<EOL><INDENT>cls = part.find('<STR_LIT:name>').text<EOL>if cls not in classes_in_person:<EOL><INDENT>continue<EOL><DEDENT>cls_id = classes.index(cls)<EOL>xmlbox = part.find('<STR_LIT>')<EOL>b = (<EOL>float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text),<EOL>float(xmlbox.find('<STR_LIT>').text), float(xmlbox.find('<STR_LIT>').text)<EOL>)<EOL>bb = convert((w, h), b)<EOL>out_file += str(cls_id) + "<STR_LIT:U+0020>" + "<STR_LIT:U+0020>".join([str(a) for a in bb]) + '<STR_LIT:\n>'<EOL>n_objs += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>in_file.close()<EOL>return n_objs, out_file<EOL><DEDENT>logging.info("<STR_LIT>")<EOL>n_objs_list = []<EOL>objs_info_list = []  <EOL>objs_info_dicts = {}<EOL>for idx, ann_file in enumerate(imgs_ann_file_list):<EOL><INDENT>n_objs, objs_info = convert_annotation(ann_file)<EOL>n_objs_list.append(n_objs)<EOL>objs_info_list.append(objs_info)<EOL>with tf.gfile.GFile(ann_file, '<STR_LIT:r>') as fid:<EOL><INDENT>xml_str = fid.read()<EOL><DEDENT>xml = etree.fromstring(xml_str)<EOL>data = _recursive_parse_xml_to_dict(xml)['<STR_LIT>']<EOL>objs_info_dicts.update({imgs_file_list[idx]: data})<EOL><DEDENT>return imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list, classes, classes_in_person, classes_dict, n_objs_list, objs_info_list, objs_info_dicts<EOL>

Pascal VOC 2007/2012 Dataset.

    It has 20 objects:
    aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow, diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor
    and additional 3 classes : head, hand, foot for person.

    Parameters
    -----------
    path : str
        The path that the data is downloaded to, defaults is ``data/VOC``.
    dataset : str
        The VOC dataset version, `2012`, `2007`, `2007test` or `2012test`. We usually train model on `2007+2012` and test it on `2007test`.
    contain_classes_in_person : boolean
        Whether include head, hand and foot annotation, default is False.

    Returns
    ---------
    imgs_file_list : list of str
        Full paths of all images.
    imgs_semseg_file_list : list of str
        Full paths of all maps for semantic segmentation. Note that not all images have this map!
    imgs_insseg_file_list : list of str
        Full paths of all maps for instance segmentation. Note that not all images have this map!
    imgs_ann_file_list : list of str
        Full paths of all annotations for bounding box and object class, all images have this annotations.
    classes : list of str
        Classes in order.
    classes_in_person : list of str
        Classes in person.
    classes_dict : dictionary
        Class label to integer.
    n_objs_list : list of int
        Number of objects in all images in ``imgs_file_list`` in order.
    objs_info_list : list of str
        Darknet format for the annotation of all images in ``imgs_file_list`` in order. ``[class_id x_centre y_centre width height]`` in ratio format.
    objs_info_dicts : dictionary
        The annotation of all images in ``imgs_file_list``, ``{imgs_file_list : dictionary for annotation}``,
        format from `TensorFlow/Models/object-detection <https://github.com/tensorflow/models/blob/master/object_detection/create_pascal_tf_record.py>`__.

    Examples
    ----------
    >>> imgs_file_list, imgs_semseg_file_list, imgs_insseg_file_list, imgs_ann_file_list,
    >>>     classes, classes_in_person, classes_dict,
    >>>     n_objs_list, objs_info_list, objs_info_dicts = tl.files.load_voc_dataset(dataset="2012", contain_classes_in_person=False)
    >>> idx = 26
    >>> print(classes)
    ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
    >>> print(classes_dict)
    {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
    >>> print(imgs_file_list[idx])
    data/VOC/VOC2012/JPEGImages/2007_000423.jpg
    >>> print(n_objs_list[idx])
    2
    >>> print(imgs_ann_file_list[idx])
    data/VOC/VOC2012/Annotations/2007_000423.xml
    >>> print(objs_info_list[idx])
    14 0.173 0.461333333333 0.142 0.496
    14 0.828 0.542666666667 0.188 0.594666666667
    >>> ann = tl.prepro.parse_darknet_ann_str_to_list(objs_info_list[idx])
    >>> print(ann)
    [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
    >>> c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
    >>> print(c, b)
    [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]

    References
    -------------
    - `Pascal VOC2012 Website <http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#devkit>`__.
    - `Pascal VOC2007 Website <http://host.robots.ox.ac.uk/pascal/VOC/voc2007/>`__.


path = os.path.join(path, '<STR_LIT>')<EOL>logging.info("<STR_LIT>".format(path))<EOL>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename = "<STR_LIT>"<EOL>if folder_exists(os.path.join(path, extracted_filename)) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(extracted_filename, path))<EOL>maybe_download_and_extract(tar_filename, path, url, extract=True)<EOL>del_file(os.path.join(path, tar_filename))<EOL><DEDENT>url = "<STR_LIT>"<EOL>tar_filename = "<STR_LIT>"<EOL>extracted_filename2 = "<STR_LIT>"<EOL>if folder_exists(os.path.join(path, extracted_filename2)) is False:<EOL><INDENT>logging.info("<STR_LIT>".format(extracted_filename, path))<EOL>maybe_download_and_extract(tar_filename, path, url, extract=True)<EOL>del_file(os.path.join(path, tar_filename))<EOL><DEDENT>logging.info("<STR_LIT>")<EOL>ann_train_list = []<EOL>ann_test_list = []<EOL>img_train_list = []<EOL>img_test_list = []<EOL>def save_joints():<EOL><INDENT>mat = sio.loadmat(os.path.join(path, extracted_filename, "<STR_LIT>"))<EOL>for _, (anno, train_flag) in enumerate(  <EOL>zip(mat['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>, <NUM_LIT:0>][<NUM_LIT:0>], mat['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>, <NUM_LIT:0>][<NUM_LIT:0>])):<EOL><INDENT>img_fn = anno['<STR_LIT:image>']['<STR_LIT:name>'][<NUM_LIT:0>, <NUM_LIT:0>][<NUM_LIT:0>]<EOL>train_flag = int(train_flag)<EOL>if train_flag:<EOL><INDENT>img_train_list.append(img_fn)<EOL>ann_train_list.append([])<EOL><DEDENT>else:<EOL><INDENT>img_test_list.append(img_fn)<EOL>ann_test_list.append([])<EOL><DEDENT>head_rect = []<EOL>if '<STR_LIT>' in str(anno['<STR_LIT>'].dtype):<EOL><INDENT>head_rect = zip(<EOL>[x1[<NUM_LIT:0>, <NUM_LIT:0>] for x1 in anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]], [y1[<NUM_LIT:0>, <NUM_LIT:0>] for y1 in anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]],<EOL>[x2[<NUM_LIT:0>, <NUM_LIT:0>] for x2 in anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]], [y2[<NUM_LIT:0>, <NUM_LIT:0>] for y2 in anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]]<EOL>)<EOL><DEDENT>else:<EOL><INDENT>head_rect = []  <EOL><DEDENT>if '<STR_LIT>' in str(anno['<STR_LIT>'].dtype):<EOL><INDENT>annopoints = anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]<EOL>head_x1s = anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]<EOL>head_y1s = anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]<EOL>head_x2s = anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]<EOL>head_y2s = anno['<STR_LIT>']['<STR_LIT>'][<NUM_LIT:0>]<EOL>for annopoint, head_x1, head_y1, head_x2, head_y2 in zip(annopoints, head_x1s, head_y1s, head_x2s,<EOL>head_y2s):<EOL><INDENT>if annopoint.size:<EOL><INDENT>head_rect = [<EOL>float(head_x1[<NUM_LIT:0>, <NUM_LIT:0>]),<EOL>float(head_y1[<NUM_LIT:0>, <NUM_LIT:0>]),<EOL>float(head_x2[<NUM_LIT:0>, <NUM_LIT:0>]),<EOL>float(head_y2[<NUM_LIT:0>, <NUM_LIT:0>])<EOL>]<EOL>annopoint = annopoint['<STR_LIT>'][<NUM_LIT:0>, <NUM_LIT:0>]<EOL>j_id = [str(j_i[<NUM_LIT:0>, <NUM_LIT:0>]) for j_i in annopoint['<STR_LIT:id>'][<NUM_LIT:0>]]<EOL>x = [x[<NUM_LIT:0>, <NUM_LIT:0>] for x in annopoint['<STR_LIT:x>'][<NUM_LIT:0>]]<EOL>y = [y[<NUM_LIT:0>, <NUM_LIT:0>] for y in annopoint['<STR_LIT:y>'][<NUM_LIT:0>]]<EOL>joint_pos = {}<EOL>for _j_id, (_x, _y) in zip(j_id, zip(x, y)):<EOL><INDENT>joint_pos[int(_j_id)] = [float(_x), float(_y)]<EOL><DEDENT>if '<STR_LIT>' in str(annopoint.dtype):<EOL><INDENT>vis = [v[<NUM_LIT:0>] if v.size > <NUM_LIT:0> else [<NUM_LIT:0>] for v in annopoint['<STR_LIT>'][<NUM_LIT:0>]]<EOL>vis = dict([(k, int(v[<NUM_LIT:0>])) if len(v) > <NUM_LIT:0> else v for k, v in zip(j_id, vis)])<EOL><DEDENT>else:<EOL><INDENT>vis = None<EOL><DEDENT>if ((is_16_pos_only ==True) and (len(joint_pos) == <NUM_LIT:16>)) or (is_16_pos_only == False):<EOL><INDENT>data = {<EOL>'<STR_LIT:filename>': img_fn,<EOL>'<STR_LIT:train>': train_flag,<EOL>'<STR_LIT>': head_rect,<EOL>'<STR_LIT>': vis,<EOL>'<STR_LIT>': joint_pos<EOL>}<EOL>if train_flag:<EOL><INDENT>ann_train_list[-<NUM_LIT:1>].append(data)<EOL><DEDENT>else:<EOL><INDENT>ann_test_list[-<NUM_LIT:1>].append(data)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>save_joints()<EOL>logging.info("<STR_LIT>")<EOL>img_dir = os.path.join(path, extracted_filename2)<EOL>_img_list = load_file_list(path=os.path.join(path, extracted_filename2), regx='<STR_LIT>', printable=False)<EOL>for i, im in enumerate(img_train_list):<EOL><INDENT>if im not in _img_list:<EOL><INDENT>print('<STR_LIT>'.format(im, img_dir))<EOL>del img_train_list[i]<EOL>del ann_train_list[i]<EOL><DEDENT><DEDENT>for i, im in enumerate(img_test_list):<EOL><INDENT>if im not in _img_list:<EOL><INDENT>print('<STR_LIT>'.format(im, img_dir))<EOL>del img_train_list[i]<EOL>del ann_train_list[i]<EOL><DEDENT><DEDENT>n_train_images = len(img_train_list)<EOL>n_test_images = len(img_test_list)<EOL>n_images = n_train_images + n_test_images<EOL>logging.info("<STR_LIT>".format(n_images, n_train_images, n_test_images))<EOL>n_train_ann = len(ann_train_list)<EOL>n_test_ann = len(ann_test_list)<EOL>n_ann = n_train_ann + n_test_ann<EOL>logging.info("<STR_LIT>".format(n_ann, n_train_ann, n_test_ann))<EOL>n_train_people = len(sum(ann_train_list, []))<EOL>n_test_people = len(sum(ann_test_list, []))<EOL>n_people = n_train_people + n_test_people<EOL>logging.info("<STR_LIT>".format(n_people, n_train_people, n_test_people))<EOL>for i, value in enumerate(img_train_list):<EOL><INDENT>img_train_list[i] = os.path.join(img_dir, value)<EOL><DEDENT>for i, value in enumerate(img_test_list):<EOL><INDENT>img_test_list[i] = os.path.join(img_dir, value)<EOL><DEDENT>return img_train_list, ann_train_list, img_test_list, ann_test_list<EOL>

Load MPII Human Pose Dataset.

    Parameters
    -----------
    path : str
        The path that the data is downloaded to.
    is_16_pos_only : boolean
        If True, only return the peoples contain 16 pose keypoints. (Usually be used for single person pose estimation)

    Returns
    ----------
    img_train_list : list of str
        The image directories of training data.
    ann_train_list : list of dict
        The annotations of training data.
    img_test_list : list of str
        The image directories of testing data.
    ann_test_list : list of dict
        The annotations of testing data.

    Examples
    --------
    >>> import pprint
    >>> import tensorlayer as tl
    >>> img_train_list, ann_train_list, img_test_list, ann_test_list = tl.files.load_mpii_pose_dataset()
    >>> image = tl.vis.read_image(img_train_list[0])
    >>> tl.vis.draw_mpii_pose_to_image(image, ann_train_list[0], 'image.png')
    >>> pprint.pprint(ann_train_list[0])

    References
    -----------
    - `MPII Human Pose Dataset. CVPR 14 <http://human-pose.mpi-inf.mpg.de>`__
    - `MPII Human Pose Models. CVPR 16 <http://pose.mpi-inf.mpg.de>`__
    - `MPII Human Shape, Poselet Conditioned Pictorial Structures and etc <http://pose.mpi-inf.mpg.de/#related>`__
    - `MPII Keyponts and ID <http://human-pose.mpi-inf.mpg.de/#download>`__


def save_npz(save_list=None, name='<STR_LIT>', sess=None):

logging.info("<STR_LIT>" % name)<EOL>if save_list is None:<EOL><INDENT>save_list = []<EOL><DEDENT>save_list_var = []<EOL>if sess:<EOL><INDENT>save_list_var = sess.run(save_list)<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>save_list_var.extend([v.eval() for v in save_list])<EOL><DEDENT>except Exception:<EOL><INDENT>logging.info(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>np.savez(name, params=save_list_var)<EOL>save_list_var = None<EOL>del save_list_var<EOL>logging.info("<STR_LIT>")<EOL>

Input parameters and the file name, save parameters into .npz file. Use tl.utils.load_npz() to restore.

    Parameters
    ----------
    save_list : list of tensor
        A list of parameters (tensor) to be saved.
    name : str
        The name of the `.npz` file.
    sess : None or Session
        Session may be required in some case.

    Examples
    --------
    Save model to npz

    >>> tl.files.save_npz(network.all_params, name='model.npz', sess=sess)

    Load model from npz (Method 1)

    >>> load_params = tl.files.load_npz(name='model.npz')
    >>> tl.files.assign_params(sess, load_params, network)

    Load model from npz (Method 2)

    >>> tl.files.load_and_assign_npz(sess=sess, name='model.npz', network=network)

    Notes
    -----
    If you got session issues, you can change the value.eval() to value.eval(session=sess)

    References
    ----------
    `Saving dictionary using numpy <http://stackoverflow.com/questions/22315595/saving-dictionary-of-header-information-using-numpy-savez>`__


d = np.load(os.path.join(path, name))<EOL>return d['<STR_LIT>']<EOL>

Load the parameters of a Model saved by tl.files.save_npz().

    Parameters
    ----------
    path : str
        Folder path to `.npz` file.
    name : str
        The name of the `.npz` file.

    Returns
    --------
    list of array
        A list of parameters in order.

    Examples
    --------
    - See ``tl.files.save_npz``

    References
    ----------
    - `Saving dictionary using numpy <http://stackoverflow.com/questions/22315595/saving-dictionary-of-header-information-using-numpy-savez>`__


def assign_params(sess, params, network):

ops = []<EOL>for idx, param in enumerate(params):<EOL><INDENT>ops.append(network.all_params[idx].assign(param))<EOL><DEDENT>if sess is not None:<EOL><INDENT>sess.run(ops)<EOL><DEDENT>return ops<EOL>

Assign the given parameters to the TensorLayer network.

    Parameters
    ----------
    sess : Session
        TensorFlow Session.
    params : list of array
        A list of parameters (array) in order.
    network : :class:`Layer`
        The network to be assigned.

    Returns
    --------
    list of operations
        A list of tf ops in order that assign params. Support sess.run(ops) manually.

    Examples
    --------
    - See ``tl.files.save_npz``

    References
    ----------
    - `Assign value to a TensorFlow variable <http://stackoverflow.com/questions/34220532/how-to-assign-value-to-a-tensorflow-variable>`__


def load_and_assign_npz(sess=None, name=None, network=None):

if network is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if sess is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not os.path.exists(name):<EOL><INDENT>logging.error("<STR_LIT>".format(name))<EOL>return False<EOL><DEDENT>else:<EOL><INDENT>params = load_npz(name=name)<EOL>assign_params(sess, params, network)<EOL>logging.info("<STR_LIT>".format(name))<EOL>return network<EOL><DEDENT>

Load model from npz and assign to a network.

    Parameters
    -------------
    sess : Session
        TensorFlow Session.
    name : str
        The name of the `.npz` file.
    network : :class:`Layer`
        The network to be assigned.

    Returns
    --------
    False or network
        Returns False, if the model is not exist.

    Examples
    --------
    - See ``tl.files.save_npz``


def save_npz_dict(save_list=None, name='<STR_LIT>', sess=None):

if sess is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if save_list is None:<EOL><INDENT>save_list = []<EOL><DEDENT>save_list_names = [tensor.name for tensor in save_list]<EOL>save_list_var = sess.run(save_list)<EOL>save_var_dict = {save_list_names[idx]: val for idx, val in enumerate(save_list_var)}<EOL>np.savez(name, **save_var_dict)<EOL>save_list_var = None<EOL>save_var_dict = None<EOL>del save_list_var<EOL>del save_var_dict<EOL>logging.info("<STR_LIT>" % name)<EOL>

Input parameters and the file name, save parameters as a dictionary into .npz file.

    Use ``tl.files.load_and_assign_npz_dict()`` to restore.

    Parameters
    ----------
    save_list : list of parameters
        A list of parameters (tensor) to be saved.
    name : str
        The name of the `.npz` file.
    sess : Session
        TensorFlow Session.


if sess is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if not os.path.exists(name):<EOL><INDENT>logging.error("<STR_LIT>".format(name))<EOL>return False<EOL><DEDENT>params = np.load(name)<EOL>if len(params.keys()) != len(set(params.keys())):<EOL><INDENT>raise Exception("<STR_LIT>" % name)<EOL><DEDENT>ops = list()<EOL>for key in params.keys():<EOL><INDENT>try:<EOL><INDENT>varlist = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=key)<EOL>if len(varlist) > <NUM_LIT:1>:<EOL><INDENT>raise Exception("<STR_LIT>" % key)<EOL><DEDENT>elif len(varlist) == <NUM_LIT:0>:<EOL><INDENT>raise KeyError<EOL><DEDENT>else:<EOL><INDENT>ops.append(varlist[<NUM_LIT:0>].assign(params[key]))<EOL>logging.info("<STR_LIT>" % key)<EOL><DEDENT><DEDENT>except KeyError:<EOL><INDENT>logging.info("<STR_LIT>" % key)<EOL><DEDENT><DEDENT>sess.run(ops)<EOL>logging.info("<STR_LIT>" % name)<EOL>

Restore the parameters saved by ``tl.files.save_npz_dict()``.

    Parameters
    ----------
    name : str
        The name of the `.npz` file.
    sess : Session
        TensorFlow Session.


def save_ckpt(<EOL>sess=None, mode_name='<STR_LIT>', save_dir='<STR_LIT>', var_list=None, global_step=None, printable=False<EOL>):

if sess is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if var_list is None:<EOL><INDENT>var_list = []<EOL><DEDENT>ckpt_file = os.path.join(save_dir, mode_name)<EOL>if var_list == []:<EOL><INDENT>var_list = tf.global_variables()<EOL><DEDENT>logging.info("<STR_LIT>" % (ckpt_file, len(var_list)))<EOL>if printable:<EOL><INDENT>for idx, v in enumerate(var_list):<EOL><INDENT>logging.info("<STR_LIT>".format(idx, v.name, str(v.get_shape())))<EOL><DEDENT><DEDENT>saver = tf.train.Saver(var_list)<EOL>saver.save(sess, ckpt_file, global_step=global_step)<EOL>

Save parameters into `ckpt` file.

    Parameters
    ------------
    sess : Session
        TensorFlow Session.
    mode_name : str
        The name of the model, default is ``model.ckpt``.
    save_dir : str
        The path / file directory to the `ckpt`, default is ``checkpoint``.
    var_list : list of tensor
        The parameters / variables (tensor) to be saved. If empty, save all global variables (default).
    global_step : int or None
        Step number.
    printable : boolean
        Whether to print all parameters information.

    See Also
    --------
    load_ckpt


if save_dict is None:<EOL><INDENT>save_dict = {}<EOL><DEDENT>np.save(name, save_dict)<EOL>

Save variables to `.npy` file.

    Parameters
    ------------
    save_dict : directory
        The variables to be saved.
    name : str
        File name.

    Examples
    ---------
    >>> tl.files.save_any_to_npy(save_dict={'data': ['a','b']}, name='test.npy')
    >>> data = tl.files.load_npy_to_any(name='test.npy')
    >>> print(data)
    {'data': ['a','b']}


file_path = os.path.join(path, name)<EOL>try:<EOL><INDENT>return np.load(file_path).item()<EOL><DEDENT>except Exception:<EOL><INDENT>return np.load(file_path)<EOL><DEDENT>raise Exception("<STR_LIT>" % file_path)<EOL>

Load `.npy` file.

    Parameters
    ------------
    path : str
        Path to the file (optional).
    name : str
        File name.

    Examples
    ---------
    - see tl.files.save_any_to_npy()


return os.path.isfile(filepath)<EOL>

Check whether a file exists by given file path.


return os.path.isdir(folderpath)<EOL>

Check whether a folder exists by given folder path.


os.remove(filepath)<EOL>


shutil.rmtree(folderpath)<EOL>


with open(filepath, '<STR_LIT:r>') as afile:<EOL><INDENT>return afile.read()<EOL><DEDENT>

Read a file and return a string.

    Examples
    ---------
    >>> data = tl.files.read_file('data.txt')


def load_file_list(path=None, regx='<STR_LIT>', printable=True, keep_prefix=False):

if path is None:<EOL><INDENT>path = os.getcwd()<EOL><DEDENT>file_list = os.listdir(path)<EOL>return_list = []<EOL>for _, f in enumerate(file_list):<EOL><INDENT>if re.search(regx, f):<EOL><INDENT>return_list.append(f)<EOL><DEDENT><DEDENT>if keep_prefix:<EOL><INDENT>for i, f in enumerate(return_list):<EOL><INDENT>return_list[i] = os.path.join(path, f)<EOL><DEDENT><DEDENT>if printable:<EOL><INDENT>logging.info('<STR_LIT>' % return_list)<EOL>logging.info('<STR_LIT>' % len(return_list))<EOL><DEDENT>return return_list<EOL>

r"""Return a file list in a folder by given a path and regular expression.

    Parameters
    ----------
    path : str or None
        A folder path, if `None`, use the current directory.
    regx : str
        The regx of file name.
    printable : boolean
        Whether to print the files infomation.
    keep_prefix : boolean
        Whether to keep path in the file name.

    Examples
    ----------
    >>> file_list = tl.files.load_file_list(path=None, regx='w1pre_[0-9]+\.(npz)')


return [os.path.join(path, o) for o in os.listdir(path) if os.path.isdir(os.path.join(path, o))]<EOL>

Return a folder list in a folder by given a folder path.

    Parameters
    ----------
    path : str
        A folder path.


if not os.path.exists(path):<EOL><INDENT>if verbose:<EOL><INDENT>logging.info("<STR_LIT>" % path)<EOL><DEDENT>os.makedirs(path)<EOL>return False<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>logging.info("<STR_LIT>" % path)<EOL><DEDENT>return True<EOL><DEDENT>

Check a folder by given name, if not exist, create the folder and return False,
    if directory exists, return True.

    Parameters
    ----------
    path : str
        A folder path.
    verbose : boolean
        If True (default), prints results.

    Returns
    --------
    boolean
        True if folder already exist, otherwise, returns False and create the folder.

    Examples
    --------
    >>> tl.files.exists_or_mkdir("checkpoints/train")


def maybe_download_and_extract(filename, working_directory, url_source, extract=False, expected_bytes=None):

<EOL>def _download(filename, working_directory, url_source):<EOL><INDENT>progress_bar = progressbar.ProgressBar()<EOL>def _dlProgress(count, blockSize, totalSize, pbar=progress_bar):<EOL><INDENT>if (totalSize != <NUM_LIT:0>):<EOL><INDENT>if not pbar.max_value:<EOL><INDENT>totalBlocks = math.ceil(float(totalSize) / float(blockSize))<EOL>pbar.max_value = int(totalBlocks)<EOL><DEDENT>pbar.update(count, force=True)<EOL><DEDENT><DEDENT>filepath = os.path.join(working_directory, filename)<EOL>logging.info('<STR_LIT>' % filename)<EOL>urlretrieve(url_source + filename, filepath, reporthook=_dlProgress)<EOL><DEDENT>exists_or_mkdir(working_directory, verbose=False)<EOL>filepath = os.path.join(working_directory, filename)<EOL>if not os.path.exists(filepath):<EOL><INDENT>_download(filename, working_directory, url_source)<EOL>statinfo = os.stat(filepath)<EOL>logging.info('<STR_LIT>' % (filename, statinfo.st_size))  <EOL>if (not (expected_bytes is None) and (expected_bytes != statinfo.st_size)):<EOL><INDENT>raise Exception('<STR_LIT>' + filename + '<STR_LIT>')<EOL><DEDENT>if (extract):<EOL><INDENT>if tarfile.is_tarfile(filepath):<EOL><INDENT>logging.info('<STR_LIT>')<EOL>tarfile.open(filepath, '<STR_LIT:r>').extractall(working_directory)<EOL>logging.info('<STR_LIT>')<EOL><DEDENT>elif zipfile.is_zipfile(filepath):<EOL><INDENT>logging.info('<STR_LIT>')<EOL>with zipfile.ZipFile(filepath) as zf:<EOL><INDENT>zf.extractall(working_directory)<EOL><DEDENT>logging.info('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>logging.info("<STR_LIT>")<EOL><DEDENT><DEDENT><DEDENT>return filepath<EOL>

Checks if file exists in working_directory otherwise tries to dowload the file,
    and optionally also tries to extract the file if format is ".zip" or ".tar"

    Parameters
    -----------
    filename : str
        The name of the (to be) dowloaded file.
    working_directory : str
        A folder path to search for the file in and dowload the file to
    url : str
        The URL to download the file from
    extract : boolean
        If True, tries to uncompress the dowloaded file is ".tar.gz/.tar.bz2" or ".zip" file, default is False.
    expected_bytes : int or None
        If set tries to verify that the downloaded file is of the specified size, otherwise raises an Exception, defaults is None which corresponds to no check being performed.

    Returns
    ----------
    str
        File path of the dowloaded (uncompressed) file.

    Examples
    --------
    >>> down_file = tl.files.maybe_download_and_extract(filename='train-images-idx3-ubyte.gz',
    ...                                            working_directory='data/',
    ...                                            url_source='http://yann.lecun.com/exdb/mnist/')
    >>> tl.files.maybe_download_and_extract(filename='ADEChallengeData2016.zip',
    ...                                             working_directory='data/',
    ...                                             url_source='http://sceneparsing.csail.mit.edu/data/',
    ...                                             extract=True)


<EOL>def atoi(text):<EOL><INDENT>return int(text) if text.isdigit() else text<EOL><DEDENT>return [atoi(c) for c in re.split('<STR_LIT>', text)]<EOL>

Sort list of string with number in human order.

    Examples
    ----------
    >>> l = ['im1.jpg', 'im31.jpg', 'im11.jpg', 'im21.jpg', 'im03.jpg', 'im05.jpg']
    >>> l.sort(key=tl.files.natural_keys)
    ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']
    >>> l.sort() # that is what we dont want
    ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']

    References
    ----------
    - `link <http://nedbatchelder.com/blog/200712/human_sorting.html>`__


file_list = load_file_list(path=path, regx=regx)<EOL>for f in file_list:<EOL><INDENT>W = load_npz(path, f)[<NUM_LIT:0>]<EOL>logging.info("<STR_LIT>" % (f, f.split('<STR_LIT:.>')[<NUM_LIT:0>] + '<STR_LIT>'))<EOL>visualize.draw_weights(W, second=<NUM_LIT:10>, saveable=True, name=f.split('<STR_LIT:.>')[<NUM_LIT:0>], fig_idx=<NUM_LIT>)<EOL><DEDENT>

r"""Convert the first weight matrix of `.npz` file to `.pdf` by using `tl.visualize.W()`.

    Parameters
    ----------
    path : str
        A folder path to `npz` files.
    regx : str
        Regx for the file name.

    Examples
    ---------
    Convert the first weight matrix of w1_pre...npz file to w1_pre...pdf.

    >>> tl.files.npz_to_W_pdf(path='/Users/.../npz_file/', regx='w1pre_[0-9]+\.(npz)')


def __init__(self, learning_rate=<NUM_LIT>, beta1=<NUM_LIT>, beta2=<NUM_LIT>, epsilon=<NUM_LIT>, use_locking=False, name="<STR_LIT>"):

super(AMSGrad, self).__init__(use_locking, name)<EOL>self._lr = learning_rate<EOL>self._beta1 = beta1<EOL>self._beta2 = beta2<EOL>self._epsilon = epsilon<EOL>self._lr_t = None<EOL>self._beta1_t = None<EOL>self._beta2_t = None<EOL>self._epsilon_t = None<EOL>self._beta1_power = None<EOL>self._beta2_power = None<EOL>


def roi_pooling(input, rois, pool_height, pool_width):

<EOL>out = roi_pooling_module.roi_pooling(input, rois, pool_height=pool_height, pool_width=pool_width)<EOL>output, argmax_output = out[<NUM_LIT:0>], out[<NUM_LIT:1>]<EOL>return output<EOL>

returns a tensorflow operation for computing the Region of Interest Pooling

@arg input: feature maps on which to perform the pooling operation
@arg rois: list of regions of interest in the format (feature map index, upper left, bottom right)
@arg pool_width: size of the pooling sections


@private_method<EOL><INDENT>def _to_bc_h_w(self, x, x_shape):<DEDENT>

x = tf.transpose(x, [<NUM_LIT:0>, <NUM_LIT:3>, <NUM_LIT:1>, <NUM_LIT:2>])<EOL>x = tf.reshape(x, (-<NUM_LIT:1>, x_shape[<NUM_LIT:1>], x_shape[<NUM_LIT:2>]))<EOL>return x<EOL>


@private_method<EOL><INDENT>def _to_b_h_w_n_c(self, x, x_shape):<DEDENT>

x = tf.reshape(x, (-<NUM_LIT:1>, x_shape[<NUM_LIT:4>], x_shape[<NUM_LIT:1>], x_shape[<NUM_LIT:2>], x_shape[<NUM_LIT:3>]))<EOL>x = tf.transpose(x, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:3>, <NUM_LIT:4>, <NUM_LIT:1>])<EOL>return x<EOL>


return tf.reshape(a, [-<NUM_LIT:1>])<EOL>


@private_method<EOL><INDENT>def _tf_repeat(self, a, repeats):<DEDENT>

<EOL>if len(a.get_shape()) != <NUM_LIT:1>:<EOL><INDENT>raise AssertionError("<STR_LIT>")<EOL><DEDENT>a = tf.expand_dims(a, -<NUM_LIT:1>)<EOL>a = tf.tile(a, [<NUM_LIT:1>, repeats])<EOL>a = self.tf_flatten(a)<EOL>return a<EOL>


@private_method<EOL><INDENT>def _tf_batch_map_coordinates(self, inputs, coords):<DEDENT>

input_shape = inputs.get_shape()<EOL>coords_shape = coords.get_shape()<EOL>batch_channel = tf.shape(inputs)[<NUM_LIT:0>]<EOL>input_h = int(input_shape[<NUM_LIT:1>])<EOL>input_w = int(input_shape[<NUM_LIT:2>])<EOL>kernel_n = int(coords_shape[<NUM_LIT:3>])<EOL>n_coords = input_h * input_w * kernel_n<EOL>coords_lt = tf.cast(tf.floor(coords), '<STR_LIT>')<EOL>coords_rb = tf.cast(tf.ceil(coords), '<STR_LIT>')<EOL>coords_lb = tf.stack([coords_lt[:, :, :, :, <NUM_LIT:0>], coords_rb[:, :, :, :, <NUM_LIT:1>]], axis=-<NUM_LIT:1>)<EOL>coords_rt = tf.stack([coords_rb[:, :, :, :, <NUM_LIT:0>], coords_lt[:, :, :, :, <NUM_LIT:1>]], axis=-<NUM_LIT:1>)<EOL>idx = self._tf_repeat(tf.range(batch_channel), n_coords)<EOL>vals_lt = self._get_vals_by_coords(inputs, coords_lt, idx, (batch_channel, input_h, input_w, kernel_n))<EOL>vals_rb = self._get_vals_by_coords(inputs, coords_rb, idx, (batch_channel, input_h, input_w, kernel_n))<EOL>vals_lb = self._get_vals_by_coords(inputs, coords_lb, idx, (batch_channel, input_h, input_w, kernel_n))<EOL>vals_rt = self._get_vals_by_coords(inputs, coords_rt, idx, (batch_channel, input_h, input_w, kernel_n))<EOL>coords_offset_lt = coords - tf.cast(coords_lt, '<STR_LIT>')<EOL>vals_t = vals_lt + (vals_rt - vals_lt) * coords_offset_lt[:, :, :, :, <NUM_LIT:0>]<EOL>vals_b = vals_lb + (vals_rb - vals_lb) * coords_offset_lt[:, :, :, :, <NUM_LIT:0>]<EOL>mapped_vals = vals_t + (vals_b - vals_t) * coords_offset_lt[:, :, :, :, <NUM_LIT:1>]<EOL>return mapped_vals<EOL>

Batch version of tf_map_coordinates

        Only supports 2D feature maps

        Parameters
        ----------
        inputs : ``tf.Tensor``
            shape = (b*c, h, w)
        coords : ``tf.Tensor``
            shape = (b*c, h, w, n, 2)

        Returns
        -------
        ``tf.Tensor``
            A Tensor with the shape as (b*c, h, w, n)


@private_method<EOL><INDENT>def _tf_batch_map_offsets(self, inputs, offsets, grid_offset):<DEDENT>

input_shape = inputs.get_shape()<EOL>batch_size = tf.shape(inputs)[<NUM_LIT:0>]<EOL>kernel_n = int(int(offsets.get_shape()[<NUM_LIT:3>]) / <NUM_LIT:2>)<EOL>input_h = input_shape[<NUM_LIT:1>]<EOL>input_w = input_shape[<NUM_LIT:2>]<EOL>channel = input_shape[<NUM_LIT:3>]<EOL>inputs = self._to_bc_h_w(inputs, input_shape)<EOL>offsets = tf.reshape(offsets, (batch_size, input_h, input_w, kernel_n, <NUM_LIT:2>))<EOL>coords = tf.expand_dims(grid_offset, <NUM_LIT:0>)  <EOL>coords = tf.tile(coords, [batch_size, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>]) + offsets  <EOL>coords = tf.stack(<EOL>[<EOL>tf.clip_by_value(coords[:, :, :, :, <NUM_LIT:0>], <NUM_LIT:0.0>, tf.cast(input_h - <NUM_LIT:1>, '<STR_LIT>')),<EOL>tf.clip_by_value(coords[:, :, :, :, <NUM_LIT:1>], <NUM_LIT:0.0>, tf.cast(input_w - <NUM_LIT:1>, '<STR_LIT>'))<EOL>], axis=-<NUM_LIT:1><EOL>)<EOL>coords = tf.tile(coords, [channel, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>])<EOL>mapped_vals = self._tf_batch_map_coordinates(inputs, coords)<EOL>mapped_vals = self._to_b_h_w_n_c(mapped_vals, [batch_size, input_h, input_w, kernel_n, channel])<EOL>return mapped_vals<EOL>

Batch map offsets into input

        Parameters
        ------------
        inputs : ``tf.Tensor``
            shape = (b, h, w, c)
        offsets: ``tf.Tensor``
            shape = (b, h, w, 2*n)
        grid_offset: `tf.Tensor``
            Offset grids shape = (h, w, n, 2)

        Returns
        -------
        ``tf.Tensor``
            A Tensor with the shape as (b, h, w, c)


Simplified version of :class:`AtrousConv1dLayer`.

    Parameters
    ----------
    prev_layer : :class:`Layer`
        Previous layer.
    n_filter : int
        The number of filters.
    filter_size : int
        The filter size.
    stride : tuple of int
        The strides: (height, width).
    dilation : int
        The filter dilation size.
    act : activation function
        The activation function of this layer.
    padding : str
        The padding algorithm type: "SAME" or "VALID".
    data_format : str
        Default is 'NWC' as it is a 1D CNN.
    W_init : initializer
        The initializer for the weight matrix.
    b_init : initializer or None
        The initializer for the bias vector. If None, skip biases.
    W_init_args : dictionary
        The arguments for the weight matrix initializer.
    b_init_args : dictionary
        The arguments for the bias vector initializer.
    name : str
        A unique layer name.

    Returns
    -------
    :class:`Layer`
        A :class:`AtrousConv1dLayer` object


channel_size = int(b.shape[<NUM_LIT:0>])<EOL>new_shape = (channel_size, <NUM_LIT:1>, <NUM_LIT:1>)<EOL>return tf.reshape(b, new_shape)<EOL>


if data_format == '<STR_LIT>':<EOL><INDENT>return x * b<EOL><DEDENT>elif data_format == '<STR_LIT>':<EOL><INDENT>return x * _to_channel_first_bias(b)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % data_format)<EOL><DEDENT>

The multiplication counter part of tf.nn.bias_add.


if data_format == '<STR_LIT>':<EOL><INDENT>return tf.add(x, b)<EOL><DEDENT>elif data_format == '<STR_LIT>':<EOL><INDENT>return tf.add(x, _to_channel_first_bias(b))<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>' % data_format)<EOL><DEDENT>

Alternative implementation of tf.nn.bias_add which is compatiable with tensorRT.


def batch_normalization(x, mean, variance, offset, scale, variance_epsilon, data_format, name=None):

with ops.name_scope(name, '<STR_LIT>', [x, mean, variance, scale, offset]):<EOL><INDENT>inv = math_ops.rsqrt(variance + variance_epsilon)<EOL>if scale is not None:<EOL><INDENT>inv *= scale<EOL><DEDENT>a = math_ops.cast(inv, x.dtype)<EOL>b = math_ops.cast(offset - mean * inv if offset is not None else -mean * inv, x.dtype)<EOL>df = {'<STR_LIT>': '<STR_LIT>', '<STR_LIT>': '<STR_LIT>'}<EOL>return _bias_add(_bias_scale(x, a, df[data_format]), b, df[data_format])<EOL><DEDENT>

Data Format aware version of tf.nn.batch_normalization.


def print_params(self, details=True, session=None):

for i, p in enumerate(self.all_params):<EOL><INDENT>if details:<EOL><INDENT>try:<EOL><INDENT>val = p.eval(session=session)<EOL>logging.info(<EOL>"<STR_LIT>".<EOL>format(i, p.name, str(val.shape), p.dtype.name, val.mean(), np.median(val), val.std())<EOL>)<EOL><DEDENT>except Exception as e:<EOL><INDENT>logging.info(str(e))<EOL>raise Exception(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logging.info("<STR_LIT>".format(i, p.name, str(p.get_shape()), p.dtype.name))<EOL><DEDENT><DEDENT>logging.info("<STR_LIT>" % self.count_params())<EOL>

Print all info of parameters in the network


for i, layer in enumerate(self.all_layers):<EOL><INDENT>logging.info(<EOL>"<STR_LIT>".format(i, layer.name, str(layer.get_shape()), layer.dtype.name)<EOL>)<EOL><DEDENT>

Print all info of layers in the network.


n_params = <NUM_LIT:0><EOL>for _i, p in enumerate(self.all_params):<EOL><INDENT>n = <NUM_LIT:1><EOL>for s in p.get_shape():<EOL><INDENT>try:<EOL><INDENT>s = int(s)<EOL><DEDENT>except Exception:<EOL><INDENT>s = <NUM_LIT:1><EOL><DEDENT>if s:<EOL><INDENT>n = n * s<EOL><DEDENT><DEDENT>n_params = n_params + n<EOL><DEDENT>return n_params<EOL>

Returns the number of parameters in the network.


_params = []<EOL>for p in self.all_params:<EOL><INDENT>if session is None:<EOL><INDENT>_params.append(p.eval())<EOL><DEDENT>else:<EOL><INDENT>_params.append(session.run(p))<EOL><DEDENT><DEDENT>return _params<EOL>

Return the parameters in a list of array.


stack = inspect.stack()<EOL>if len(stack) < skip + <NUM_LIT:1>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>args, _, _, values = inspect.getargvalues(stack[skip][<NUM_LIT:0>])<EOL>params = {}<EOL>for arg in args:<EOL><INDENT>if values[arg] is not None and arg not in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>val = values[arg]<EOL>if inspect.isfunction(val):<EOL><INDENT>params[arg] = {"<STR_LIT>": val.__module__, "<STR_LIT>": val.__name__}<EOL><DEDENT>elif arg.endswith('<STR_LIT>'):<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>params[arg] = val<EOL><DEDENT><DEDENT><DEDENT>return params<EOL>

Get all arguments of current layer for saving the graph.


def transformer(U, theta, out_size, name='<STR_LIT>'):

def _repeat(x, n_repeats):<EOL><INDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>rep = tf.transpose(tf.expand_dims(tf.ones(shape=tf.stack([<EOL>n_repeats,<EOL>])), <NUM_LIT:1>), [<NUM_LIT:1>, <NUM_LIT:0>])<EOL>rep = tf.cast(rep, '<STR_LIT>')<EOL>x = tf.matmul(tf.reshape(x, (-<NUM_LIT:1>, <NUM_LIT:1>)), rep)<EOL>return tf.reshape(x, [-<NUM_LIT:1>])<EOL><DEDENT><DEDENT>def _interpolate(im, x, y, out_size):<EOL><INDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>num_batch = tf.shape(im)[<NUM_LIT:0>]<EOL>height = tf.shape(im)[<NUM_LIT:1>]<EOL>width = tf.shape(im)[<NUM_LIT:2>]<EOL>channels = tf.shape(im)[<NUM_LIT:3>]<EOL>x = tf.cast(x, '<STR_LIT>')<EOL>y = tf.cast(y, '<STR_LIT>')<EOL>height_f = tf.cast(height, '<STR_LIT>')<EOL>width_f = tf.cast(width, '<STR_LIT>')<EOL>out_height = out_size[<NUM_LIT:0>]<EOL>out_width = out_size[<NUM_LIT:1>]<EOL>zero = tf.zeros([], dtype='<STR_LIT>')<EOL>max_y = tf.cast(tf.shape(im)[<NUM_LIT:1>] - <NUM_LIT:1>, '<STR_LIT>')<EOL>max_x = tf.cast(tf.shape(im)[<NUM_LIT:2>] - <NUM_LIT:1>, '<STR_LIT>')<EOL>x = (x + <NUM_LIT:1.0>) * (width_f) / <NUM_LIT><EOL>y = (y + <NUM_LIT:1.0>) * (height_f) / <NUM_LIT><EOL>x0 = tf.cast(tf.floor(x), '<STR_LIT>')<EOL>x1 = x0 + <NUM_LIT:1><EOL>y0 = tf.cast(tf.floor(y), '<STR_LIT>')<EOL>y1 = y0 + <NUM_LIT:1><EOL>x0 = tf.clip_by_value(x0, zero, max_x)<EOL>x1 = tf.clip_by_value(x1, zero, max_x)<EOL>y0 = tf.clip_by_value(y0, zero, max_y)<EOL>y1 = tf.clip_by_value(y1, zero, max_y)<EOL>dim2 = width<EOL>dim1 = width * height<EOL>base = _repeat(tf.range(num_batch) * dim1, out_height * out_width)<EOL>base_y0 = base + y0 * dim2<EOL>base_y1 = base + y1 * dim2<EOL>idx_a = base_y0 + x0<EOL>idx_b = base_y1 + x0<EOL>idx_c = base_y0 + x1<EOL>idx_d = base_y1 + x1<EOL>im_flat = tf.reshape(im, tf.stack([-<NUM_LIT:1>, channels]))<EOL>im_flat = tf.cast(im_flat, '<STR_LIT>')<EOL>Ia = tf.gather(im_flat, idx_a)<EOL>Ib = tf.gather(im_flat, idx_b)<EOL>Ic = tf.gather(im_flat, idx_c)<EOL>Id = tf.gather(im_flat, idx_d)<EOL>x0_f = tf.cast(x0, '<STR_LIT>')<EOL>x1_f = tf.cast(x1, '<STR_LIT>')<EOL>y0_f = tf.cast(y0, '<STR_LIT>')<EOL>y1_f = tf.cast(y1, '<STR_LIT>')<EOL>wa = tf.expand_dims(((x1_f - x) * (y1_f - y)), <NUM_LIT:1>)<EOL>wb = tf.expand_dims(((x1_f - x) * (y - y0_f)), <NUM_LIT:1>)<EOL>wc = tf.expand_dims(((x - x0_f) * (y1_f - y)), <NUM_LIT:1>)<EOL>wd = tf.expand_dims(((x - x0_f) * (y - y0_f)), <NUM_LIT:1>)<EOL>output = tf.add_n([wa * Ia, wb * Ib, wc * Ic, wd * Id])<EOL>return output<EOL><DEDENT><DEDENT>def _meshgrid(height, width):<EOL><INDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>x_t = tf.matmul(<EOL>tf.ones(shape=tf.stack([height, <NUM_LIT:1>])),<EOL>tf.transpose(tf.expand_dims(tf.linspace(-<NUM_LIT:1.0>, <NUM_LIT:1.0>, width), <NUM_LIT:1>), [<NUM_LIT:1>, <NUM_LIT:0>])<EOL>)<EOL>y_t = tf.matmul(tf.expand_dims(tf.linspace(-<NUM_LIT:1.0>, <NUM_LIT:1.0>, height), <NUM_LIT:1>), tf.ones(shape=tf.stack([<NUM_LIT:1>, width])))<EOL>x_t_flat = tf.reshape(x_t, (<NUM_LIT:1>, -<NUM_LIT:1>))<EOL>y_t_flat = tf.reshape(y_t, (<NUM_LIT:1>, -<NUM_LIT:1>))<EOL>ones = tf.ones_like(x_t_flat)<EOL>grid = tf.concat(axis=<NUM_LIT:0>, values=[x_t_flat, y_t_flat, ones])<EOL>return grid<EOL><DEDENT><DEDENT>def _transform(theta, input_dim, out_size):<EOL><INDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>num_batch = tf.shape(input_dim)[<NUM_LIT:0>]<EOL>num_channels = tf.shape(input_dim)[<NUM_LIT:3>]<EOL>theta = tf.reshape(theta, (-<NUM_LIT:1>, <NUM_LIT:2>, <NUM_LIT:3>))<EOL>theta = tf.cast(theta, '<STR_LIT>')<EOL>out_height = out_size[<NUM_LIT:0>]<EOL>out_width = out_size[<NUM_LIT:1>]<EOL>grid = _meshgrid(out_height, out_width)<EOL>grid = tf.expand_dims(grid, <NUM_LIT:0>)<EOL>grid = tf.reshape(grid, [-<NUM_LIT:1>])<EOL>grid = tf.tile(grid, tf.stack([num_batch]))<EOL>grid = tf.reshape(grid, tf.stack([num_batch, <NUM_LIT:3>, -<NUM_LIT:1>]))<EOL>T_g = tf.matmul(theta, grid)<EOL>x_s = tf.slice(T_g, [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>], [-<NUM_LIT:1>, <NUM_LIT:1>, -<NUM_LIT:1>])<EOL>y_s = tf.slice(T_g, [<NUM_LIT:0>, <NUM_LIT:1>, <NUM_LIT:0>], [-<NUM_LIT:1>, <NUM_LIT:1>, -<NUM_LIT:1>])<EOL>x_s_flat = tf.reshape(x_s, [-<NUM_LIT:1>])<EOL>y_s_flat = tf.reshape(y_s, [-<NUM_LIT:1>])<EOL>input_transformed = _interpolate(input_dim, x_s_flat, y_s_flat, out_size)<EOL>output = tf.reshape(input_transformed, tf.stack([num_batch, out_height, out_width, num_channels]))<EOL>return output<EOL><DEDENT><DEDENT>with tf.variable_scope(name):<EOL><INDENT>output = _transform(theta, U, out_size)<EOL>return output<EOL><DEDENT>

Spatial Transformer Layer for `2D Affine Transformation <https://en.wikipedia.org/wiki/Affine_transformation>`__
    , see :class:`SpatialTransformer2dAffineLayer` class.

    Parameters
    ----------
    U : list of float
        The output of a convolutional net should have the
        shape [num_batch, height, width, num_channels].
    theta: float
        The output of the localisation network should be [num_batch, 6], value range should be [0, 1] (via tanh).
    out_size: tuple of int
        The size of the output of the network (height, width)
    name: str
        Optional function name

    Returns
    -------
    Tensor
        The transformed tensor.

    References
    ----------
    - `Spatial Transformer Networks <https://arxiv.org/abs/1506.02025>`__
    - `TensorFlow/Models <https://github.com/tensorflow/models/tree/master/transformer>`__

    Notes
    -----
    To initialize the network to the identity transform init.

    >>> import tensorflow as tf
    >>> # ``theta`` to
    >>> identity = np.array([[1., 0., 0.], [0., 1., 0.]])
    >>> identity = identity.flatten()
    >>> theta = tf.Variable(initial_value=identity)


def batch_transformer(U, thetas, out_size, name='<STR_LIT>'):

with tf.variable_scope(name):<EOL><INDENT>num_batch, num_transforms = map(int, thetas.get_shape().as_list()[:<NUM_LIT:2>])<EOL>indices = [[i] * num_transforms for i in xrange(num_batch)]<EOL>input_repeated = tf.gather(U, tf.reshape(indices, [-<NUM_LIT:1>]))<EOL>return transformer(input_repeated, thetas, out_size)<EOL><DEDENT>

Batch Spatial Transformer function for `2D Affine Transformation <https://en.wikipedia.org/wiki/Affine_transformation>`__.

    Parameters
    ----------
    U : list of float
        tensor of inputs [batch, height, width, num_channels]
    thetas : list of float
        a set of transformations for each input [batch, num_transforms, 6]
    out_size : list of int
        the size of the output [out_height, out_width]
    name : str
        optional function name

    Returns
    ------
    float
        Tensor of size [batch * num_transforms, out_height, out_width, num_channels]


threshold = _compute_threshold(x)<EOL>alpha1_temp1 = tf.where(tf.greater(x, threshold), x, tf.zeros_like(x, tf.float32))<EOL>alpha1_temp2 = tf.where(tf.less(x, -threshold), x, tf.zeros_like(x, tf.float32))<EOL>alpha_array = tf.add(alpha1_temp1, alpha1_temp2, name=None)<EOL>alpha_array_abs = tf.abs(alpha_array)<EOL>alpha_array_abs1 = tf.where(<EOL>tf.greater(alpha_array_abs, <NUM_LIT:0>), tf.ones_like(alpha_array_abs, tf.float32),<EOL>tf.zeros_like(alpha_array_abs, tf.float32)<EOL>)<EOL>alpha_sum = tf.reduce_sum(alpha_array_abs)<EOL>n = tf.reduce_sum(alpha_array_abs1)<EOL>alpha = tf.div(alpha_sum, n)<EOL>return alpha<EOL>


dim = <NUM_LIT:1><EOL>for d in variable.get_shape()[<NUM_LIT:1>:].as_list():<EOL><INDENT>dim *= d<EOL><DEDENT>return tf.reshape(variable, shape=[-<NUM_LIT:1>, dim], name=name)<EOL>

Reshapes a high-dimension vector input.

    [batch_size, mask_row, mask_col, n_mask] ---> [batch_size, mask_row x mask_col x n_mask]

    Parameters
    ----------
    variable : TensorFlow variable or tensor
        The variable or tensor to be flatten.
    name : str
        A unique layer name.

    Returns
    -------
    Tensor
        Flatten Tensor

    Examples
    --------
    >>> import tensorflow as tf
    >>> import tensorlayer as tl
    >>> x = tf.placeholder(tf.float32, [None, 128, 128, 3])
    >>> # Convolution Layer with 32 filters and a kernel size of 5
    >>> network = tf.layers.conv2d(x, 32, 5, activation=tf.nn.relu)
    >>> # Max Pooling (down-sampling) with strides of 2 and kernel size of 2
    >>> network = tf.layers.max_pooling2d(network, 2, 2)
    >>> print(network.get_shape()[:].as_list())
    >>> [None, 62, 62, 32]
    >>> network = tl.layers.flatten_reshape(network)
    >>> print(network.get_shape()[:].as_list()[1:])
    >>> [None, 123008]


@deprecated_alias(printable='<STR_LIT>', end_support_version=<NUM_LIT>)  <EOL>def get_layers_with_name(net, name="<STR_LIT>", verbose=False):

logging.info("<STR_LIT>" % name)<EOL>layers = []<EOL>i = <NUM_LIT:0><EOL>for layer in net.all_layers:<EOL><INDENT>if name in layer.name:<EOL><INDENT>layers.append(layer)<EOL>if verbose:<EOL><INDENT>logging.info("<STR_LIT>".format(i, layer.name, str(layer.get_shape())))<EOL>i = i + <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return layers<EOL>

Get a list of layers' output in a network by a given name scope.

    Parameters
    -----------
    net : :class:`Layer`
        The last layer of the network.
    name : str
        Get the layers' output that contain this name.
    verbose : boolean
        If True, print information of all the layers' output

    Returns
    --------
    list of Tensor
        A list of layers' output (TensorFlow tensor)

    Examples
    ---------
    >>> import tensorlayer as tl
    >>> layers = tl.layers.get_layers_with_name(net, "CNN", True)


@deprecated_alias(printable='<STR_LIT>', end_support_version=<NUM_LIT>)  <EOL>def get_variables_with_name(name=None, train_only=True, verbose=False):

if name is None:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>logging.info("<STR_LIT>" % name)<EOL>if train_only:<EOL><INDENT>t_vars = tf.trainable_variables()<EOL><DEDENT>else:<EOL><INDENT>t_vars = tf.global_variables()<EOL><DEDENT>d_vars = [var for var in t_vars if name in var.name]<EOL>if verbose:<EOL><INDENT>for idx, v in enumerate(d_vars):<EOL><INDENT>logging.info("<STR_LIT>".format(idx, v.name, str(v.get_shape())))<EOL><DEDENT><DEDENT>return d_vars<EOL>

Get a list of TensorFlow variables by a given name scope.

    Parameters
    ----------
    name : str
        Get the variables that contain this name.
    train_only : boolean
        If Ture, only get the trainable variables.
    verbose : boolean
        If True, print the information of all variables.

    Returns
    -------
    list of Tensor
        A list of TensorFlow variables

    Examples
    --------
    >>> import tensorlayer as tl
    >>> dense_vars = tl.layers.get_variables_with_name('dense', True, True)


if sess is None:<EOL><INDENT>raise AssertionError('<STR_LIT>')<EOL><DEDENT>sess.run(tf.global_variables_initializer())<EOL>

Initialize the global variables of TensorFlow.

    Run ``sess.run(tf.global_variables_initializer())`` for TF 0.12+ or
    ``sess.run(tf.initialize_all_variables())`` for TF 0.11.

    Parameters
    ----------
    sess : Session
        TensorFlow session.


if isinstance(state, LSTMStateTuple):<EOL><INDENT>c = state.c.eval(feed_dict=feed_dict)<EOL>h = state.h.eval(feed_dict=feed_dict)<EOL>return c, h<EOL><DEDENT>else:<EOL><INDENT>new_state = state.eval(feed_dict=feed_dict)<EOL>return new_state<EOL><DEDENT>

Returns the initialized RNN state.
    The inputs are `LSTMStateTuple` or `State` of `RNNCells`, and an optional `feed_dict`.

    Parameters
    ----------
    state : RNN state.
        The TensorFlow's RNN state.
    feed_dict : dictionary
        Initial RNN state; if None, returns zero state.

    Returns
    -------
    RNN state
        The TensorFlow's RNN state.


y = []<EOL>for i in x:<EOL><INDENT>if i not in y:<EOL><INDENT>y.append(i)<EOL><DEDENT><DEDENT>return y<EOL>

Remove the repeated items in a list, and return the processed list.
    You may need it to create merged layer like Concat, Elementwise and etc.

    Parameters
    ----------
    x : list
        Input

    Returns
    -------
    list
        A list that after removing it's repeated items

    Examples
    -------
    >>> l = [2, 3, 4, 2, 3]
    >>> l = list_remove_repeat(l)
    [2, 3, 4]


if layers is None:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>layer = layers[<NUM_LIT:0>]<EOL>all_params = []<EOL>all_layers = []<EOL>all_drop = {}<EOL>for l in layers:<EOL><INDENT>all_params.extend(l.all_params)<EOL>all_layers.extend(l.all_layers)<EOL>all_drop.update(l.all_drop)<EOL><DEDENT>layer.all_params = list(all_params)<EOL>layer.all_layers = list(all_layers)<EOL>layer.all_drop = dict(all_drop)<EOL>layer.all_layers = list_remove_repeat(layer.all_layers)<EOL>layer.all_params = list_remove_repeat(layer.all_params)<EOL>return layer<EOL>

Merge all parameters, layers and dropout probabilities to a :class:`Layer`.
    The output of return network is the first network in the list.

    Parameters
    ----------
    layers : list of :class:`Layer`
        Merge all parameters, layers and dropout probabilities to the first layer in the list.

    Returns
    --------
    :class:`Layer`
        The network after merging all parameters, layers and dropout probabilities to the first network in the list.

    Examples
    ---------
    >>> import tensorlayer as tl
    >>> n1 = ...
    >>> n2 = ...
    >>> n1 = tl.layers.merge_networks([n1, n2])


<EOL>if train_only:<EOL><INDENT>t_vars = tf.trainable_variables()<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>t_vars = tf.global_variables()<EOL>logging.info("<STR_LIT>")<EOL><DEDENT>for idx, v in enumerate(t_vars):<EOL><INDENT>logging.info("<STR_LIT>".format(idx, str(v.get_shape()), v.name))<EOL><DEDENT>

Print information of trainable or all variables,
    without ``tl.layers.initialize_global_variables(sess)``.

    Parameters
    ----------
    train_only : boolean
        Whether print trainable variables only.
            - If True, print the trainable variables.
            - If False, print all variables.