got10k/datasets/vid.py
class ImageNetVID(object):
    r"""`ImageNet Video Image Detection (VID) <https://image-net.org/challenges/LSVRC/2015/#vid>`_ Dataset.

    Publication:
        ``ImageNet Large Scale Visual Recognition Challenge``, O. Russakovsky,
            J. deng, H. Su, etc. IJCV, 2015.
    
    Args:
        root_dir (string): Root directory of dataset where ``Data``, and
            ``Annotation`` folders exist.
        subset (string, optional): Specify ``train``, ``val`` or (``train``, ``val``)
            subset(s) of ImageNet-VID. Default is a tuple (``train``, ``val``).
        cache_dir (string, optional): Directory for caching the paths and annotations
            for speeding up loading. Default is ``cache/imagenet_vid``.
    """
    def __init__(self, root_dir, subset=('train', 'val'),
                 cache_dir='cache/imagenet_vid'):
        self.root_dir = root_dir
        self.cache_dir = cache_dir
        if isinstance(subset, str):
            assert subset in ['train', 'val']
            self.subset = [subset]
        elif isinstance(subset, (list, tuple)):
            assert all([s in ['train', 'val'] for s in subset])
            self.subset = subset
        else:
            raise Exception('Unknown subset')
        
        # cache filenames and annotations to speed up training
        self.seq_dict = self._cache_meta()
        self.seq_names = [n for n in self.seq_dict]

    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            seq_name = index
        else:
            seq_name = self.seq_names[index]

        seq_dir, frames, anno_file = self.seq_dict[seq_name]
        img_files = [os.path.join(
            seq_dir, '%06d.JPEG' % f) for f in frames]
        anno = np.loadtxt(anno_file, delimiter=',')

        return img_files, anno

    def __len__(self):
        return len(self.seq_dict)

    def _cache_meta(self):
        cache_file = os.path.join(self.cache_dir, 'seq_dict.json')
        if os.path.isfile(cache_file):
            print('Dataset already cached.')
            with open(cache_file) as f:
                seq_dict = json.load(f, object_pairs_hook=OrderedDict)
            return seq_dict
        
        # image and annotation paths
        print('Gather sequence paths...')
        seq_dirs = []
        anno_dirs = []
        if 'train' in self.subset:
            seq_dirs_ = sorted(glob.glob(os.path.join(
                self.root_dir, 'Data/VID/train/ILSVRC*/ILSVRC*')))
            anno_dirs_ = [os.path.join(
                self.root_dir, 'Annotations/VID/train',
                *s.split('/')[-2:]) for s in seq_dirs_]
            seq_dirs += seq_dirs_
            anno_dirs += anno_dirs_
        if 'val' in self.subset:
            seq_dirs_ = sorted(glob.glob(os.path.join(
                self.root_dir, 'Data/VID/val/ILSVRC2015_val_*')))
            anno_dirs_ = [os.path.join(
                self.root_dir, 'Annotations/VID/val',
                s.split('/')[-1]) for s in seq_dirs_]
            seq_dirs += seq_dirs_
            anno_dirs += anno_dirs_
        seq_names = [os.path.basename(s) for s in seq_dirs]

        # cache paths and annotations
        print('Caching annotations to %s, ' % self.cache_dir + \
            'it may take a few minutes...')
        seq_dict = OrderedDict()
        cache_anno_dir = os.path.join(self.cache_dir, 'anno')
        if not os.path.isdir(cache_anno_dir):
            os.makedirs(cache_anno_dir)

        for s, seq_name in enumerate(seq_names):
            if s % 100 == 0 or s == len(seq_names) - 1:
                print('--Caching sequence %d/%d: %s' % \
                    (s + 1, len(seq_names), seq_name))
            anno_files = sorted(glob.glob(os.path.join(
                anno_dirs[s], '*.xml')))
            objects = [ET.ElementTree(file=f).findall('object')
                       for f in anno_files]
            
            # find all track ids
            track_ids, counts = np.unique([
                obj.find('trackid').text for group in objects
                for obj in group], return_counts=True)
            
            # fetch paths and annotations for each track id
            for t, track_id in enumerate(track_ids):
                if counts[t] < 2:
                    continue
                frames = []
                anno = []
                for f, group in enumerate(objects):
                    for obj in group:
                        if not obj.find('trackid').text == track_id:
                            continue
                        frames.append(f)
                        anno.append([
                            int(obj.find('bndbox/xmin').text),
                            int(obj.find('bndbox/ymin').text),
                            int(obj.find('bndbox/xmax').text),
                            int(obj.find('bndbox/ymax').text)])
                anno = np.array(anno, dtype=int)
                anno[:, 2:] -= anno[:, :2] - 1

                # store annotations
                key = '%s.%d' % (seq_name, int(track_id))
                cache_anno_file = os.path.join(cache_anno_dir, key + '.txt')
                np.savetxt(cache_anno_file, anno, fmt='%d', delimiter=',')

                # store paths
                seq_dict.update([(key, [
                    seq_dirs[s], frames, cache_anno_file])])
        
        # store seq_dict
        with open(cache_file, 'w') as f:
            json.dump(seq_dict, f)

        return seq_dict
got10k/datasets/tcolor128.py
class TColor128(object):
    """`TColor128 <http://www.dabi.temple.edu/~hbling/data/TColor-128/TColor-128.html>`_ Dataset.

    Publication:
        ``Encoding color information for visual tracking: algorithms and benchmark``,
        P. Liang, E. Blasch and H. Ling, TIP, 2015.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
    """
    def __init__(self, root_dir, download=True):
        super(TColor128, self).__init__()
        self.root_dir = root_dir
        if download:
            self._download(root_dir)
        self._check_integrity(root_dir)

        self.anno_files = sorted(glob.glob(
            os.path.join(root_dir, '*/*_gt.txt')))
        self.seq_dirs = [os.path.dirname(f) for f in self.anno_files]
        self.seq_names = [os.path.basename(d) for d in self.seq_dirs]
        # valid frame range for each sequence
        self.range_files = [glob.glob(
            os.path.join(d, '*_frames.txt'))[0]
            for d in self.seq_dirs]
    
    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        # load valid frame range
        frames = np.loadtxt(
            self.range_files[index], dtype=int, delimiter=',')
        img_files = [os.path.join(
            self.seq_dirs[index], 'img/%04d.jpg' % f)
            for f in range(frames[0], frames[1] + 1)]

        # load annotations
        anno = np.loadtxt(self.anno_files[index], delimiter=',')
        assert len(img_files) == len(anno)
        assert anno.shape[1] == 4

        return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _download(self, root_dir):
        if not os.path.isdir(root_dir):
            os.makedirs(root_dir)
        elif len(os.listdir(root_dir)) > 100:
            print('Files already downloaded.')
            return

        url = 'http://www.dabi.temple.edu/~hbling/data/TColor-128/Temple-color-128.zip'
        zip_file = os.path.join(root_dir, 'Temple-color-128.zip')
        print('Downloading to %s...' % zip_file)
        download(url, zip_file)
        print('\nExtracting to %s...' % root_dir)
        extract(zip_file, root_dir)

        return root_dir

    def _check_integrity(self, root_dir):
        seq_names = os.listdir(root_dir)
        seq_names = [n for n in seq_names if not n[0] == '.']

        if os.path.isdir(root_dir) and len(seq_names) > 0:
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted. ' +
                            'You can use download=True to download it.')
got10k/datasets/lasot.py
class LaSOT(object):
    r"""`LaSOT <https://cis.temple.edu/lasot/>`_ Datasets.

    Publication:
        ``LaSOT: A High-quality Benchmark for Large-scale Single Object Tracking``,
        H. Fan, L. Lin, F. Yang, P. Chu, G. Deng, S. Yu, H. Bai,
        Y. Xu, C. Liao, and H. Ling., CVPR 2019.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
        subset (string, optional): Specify ``train`` or ``test``
            subset of LaSOT.
    """
    def __init__(self, root_dir, subset='test', return_meta=False):
        super(LaSOT, self).__init__()
        subset = subset.split('_')
        assert set(subset).issubset({'train', 'test'}), 'Unknown subset.'

        self.root_dir = root_dir
        self.subset = subset
        self.return_meta = return_meta
        self._check_integrity(root_dir)

        self.anno_files = sorted(glob.glob(
            os.path.join(root_dir, '*/*/groundtruth.txt')))
        self.seq_dirs = [os.path.join(
            os.path.dirname(f), 'img') for f in self.anno_files]
        self.seq_names = [os.path.basename(
            os.path.dirname(f)) for f in self.anno_files]
        
        # load subset sequence names
        split_file = os.path.join(
            os.path.dirname(__file__), 'lasot.json')
        with open(split_file, 'r') as f:
            splits = json.load(f)
        self.seq_names = []
        for s in subset:
            self.seq_names.extend(splits[s])

        # image and annotation paths
        self.seq_dirs = [os.path.join(
            root_dir, n[:n.rfind('-')], n, 'img')
            for n in self.seq_names]
        self.anno_files = [os.path.join(
            os.path.dirname(d), 'groundtruth.txt')
            for d in self.seq_dirs]

    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno) if ``return_meta`` is False, otherwise
                (img_files, anno, meta), where ``img_files`` is a list of
                file names, ``anno`` is a N x 4 (rectangles) numpy array, while
                ``meta`` is a dict contains meta information about the sequence.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        img_files = sorted(glob.glob(os.path.join(
            self.seq_dirs[index], '*.jpg')))
        anno = np.loadtxt(self.anno_files[index], delimiter=',')

        if self.return_meta:
            meta = self._fetch_meta(self.seq_dirs[index])
            return img_files, anno, meta
        else:
            return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _check_integrity(self, root_dir):
        seq_names = os.listdir(root_dir)
        seq_names = [n for n in seq_names if not n[0] == '.']

        if os.path.isdir(root_dir) and len(seq_names) > 0:
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted.')

    def _fetch_meta(self, seq_dir):
        seq_dir = os.path.dirname(seq_dir)
        meta = {}

        # attributes
        for att in ['full_occlusion', 'out_of_view']:
            att_file = os.path.join(seq_dir, att + '.txt')
            meta[att] = np.loadtxt(att_file, delimiter=',')
        
        # nlp
        nlp_file = os.path.join(seq_dir, 'nlp.txt')
        with open(nlp_file, 'r') as f:
            meta['nlp'] = f.read().strip()

        return meta
got10k/datasets/nfs.py
class NfS(object):
    """`NfS <http://ci2cv.net/nfs/index.html>`_ Dataset.

    Publication:
        ``Need for Speed: A Benchmark for Higher Frame Rate Object Tracking``,
        H. K. Galoogahi, A. Fagg, C. Huang, D. Ramanan and S. Lucey, ICCV 2017.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
        fps (integer): Sequence frame rate. Two options ``30`` and ``240``
            are available. Default is 240.
    """
    def __init__(self, root_dir, fps=240):
        super(NfS, self).__init__()
        assert fps in [30, 240]
        self.fps = fps
        self.root_dir = root_dir
        self._check_integrity(root_dir)

        self.anno_files = sorted(glob.glob(
            os.path.join(root_dir, '*/%d/*.txt' % fps)))
        self.seq_names = [
            os.path.basename(f)[:-4] for f in self.anno_files]
        self.seq_dirs = [os.path.join(
            os.path.dirname(f), n)
            for f, n in zip(self.anno_files, self.seq_names)]
    
    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        img_files = sorted(glob.glob(
            os.path.join(self.seq_dirs[index], '*.jpg')))
        anno = np.loadtxt(self.anno_files[index], dtype=str)
        anno = anno[:, 1:5].astype(float)  # [left, top, right, bottom]
        anno[:, 2:] -= anno[:, :2]         # [left, top, width, height]

        # handle inconsistent lengths
        if not len(img_files) == len(anno):
            if abs(len(anno) / len(img_files) - 8) < 1:
                anno = anno[0::8, :]
            diff = abs(len(img_files) - len(anno))
            if diff > 0 and diff <= 1:
                n = min(len(img_files), len(anno))
                anno = anno[:n]
                img_files = img_files[:n]
        assert len(img_files) == len(anno)

        return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _check_integrity(self, root_dir):
        seq_names = os.listdir(root_dir)
        seq_names = [n for n in seq_names if not n[0] == '.']

        if os.path.isdir(root_dir) and len(seq_names) > 0:
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted.')
got10k/datasets/got10k.py
class GOT10k(object):
    r"""`GOT-10K <http://got-10k.aitestunion.com//>`_ Dataset.

    Publication:
        ``GOT-10k: A Large High-Diversity Benchmark for Generic Object
        Tracking in the Wild``, L. Huang, X. Zhao and K. Huang, ArXiv 2018.
    
    Args:
        root_dir (string): Root directory of dataset where ``train``,
            ``val`` and ``test`` folders exist.
        subset (string, optional): Specify ``train``, ``val`` or ``test``
            subset of GOT-10k.
        return_meta (string, optional): If True, returns ``meta``
            of each sequence in ``__getitem__`` function, otherwise
            only returns ``img_files`` and ``anno``.
        list_file (string, optional): If provided, only read sequences
            specified by the file instead of all sequences in the subset.
    """
    def __init__(self, root_dir, subset='test', return_meta=False,
                 list_file=None, check_integrity=True):
        super(GOT10k, self).__init__()
        assert subset in ['train', 'val', 'test'], 'Unknown subset.'
        self.root_dir = root_dir
        self.subset = subset
        self.return_meta = False if subset == 'test' else return_meta
        
        if list_file is None:
            list_file = os.path.join(root_dir, subset, 'list.txt')
        if check_integrity:
            self._check_integrity(root_dir, subset, list_file)

        with open(list_file, 'r') as f:
            self.seq_names = f.read().strip().split('\n')
        self.seq_dirs = [os.path.join(root_dir, subset, s)
                         for s in self.seq_names]
        self.anno_files = [os.path.join(d, 'groundtruth.txt')
                           for d in self.seq_dirs]
    
    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno) if ``return_meta`` is False, otherwise
                (img_files, anno, meta), where ``img_files`` is a list of
                file names, ``anno`` is a N x 4 (rectangles) numpy array, while
                ``meta`` is a dict contains meta information about the sequence.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        img_files = sorted(glob.glob(os.path.join(
            self.seq_dirs[index], '*.jpg')))
        anno = np.loadtxt(self.anno_files[index], delimiter=',')

        if self.subset == 'test' and anno.ndim == 1:
            assert len(anno) == 4
            anno = anno[np.newaxis, :]
        else:
            assert len(img_files) == len(anno)

        if self.return_meta:
            meta = self._fetch_meta(self.seq_dirs[index])
            return img_files, anno, meta
        else:
            return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _check_integrity(self, root_dir, subset, list_file=None):
        assert subset in ['train', 'val', 'test']
        if list_file is None:
            list_file = os.path.join(root_dir, subset, 'list.txt')

        if os.path.isfile(list_file):
            with open(list_file, 'r') as f:
                seq_names = f.read().strip().split('\n')
            
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, subset, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted.')

    def _fetch_meta(self, seq_dir):
        # meta information
        meta_file = os.path.join(seq_dir, 'meta_info.ini')
        with open(meta_file) as f:
            meta = f.read().strip().split('\n')[1:]
        meta = [line.split(': ') for line in meta]
        meta = {line[0]: line[1] for line in meta}

        # attributes
        attributes = ['cover', 'absence', 'cut_by_image']
        for att in attributes:
            meta[att] = np.loadtxt(os.path.join(seq_dir, att + '.label'))

        return meta
got10k/datasets/vot.py
class VOT(object):
    r"""`VOT <http://www.votchallenge.net/>`_ Datasets.

    Publication:
        ``The Visual Object Tracking VOT2017 challenge results``, M. Kristan, A. Leonardis
            and J. Matas, etc. 2017.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
        version (integer, optional): Specify the benchmark version. Specify as
            one of 2013~2018. Default is 2017.
        anno_type (string, optional): Returned annotation types, chosen as one of
            ``rect`` and ``corner``. Default is ``rect``.
        download (boolean, optional): If True, downloads the dataset from the internet
            and puts it in root directory. If dataset is downloaded, it is not
            downloaded again.
        return_meta (string, optional): If True, returns ``meta``
            of each sequence in ``__getitem__`` function, otherwise
            only returns ``img_files`` and ``anno``.
        list_file (string, optional): If provided, only read sequences
            specified by the file.
    """
    __valid_versions = [2013, 2014, 2015, 2016, 2017, 2018, 'LT2018',
                        2019, 'LT2019', 'RGBD2019', 'RGBT2019']

    def __init__(self, root_dir, version=2017, anno_type='rect',
                 download=True, return_meta=False, list_file=None):
        super(VOT, self).__init__()
        assert version in self.__valid_versions, 'Unsupport VOT version.'
        assert anno_type in ['default', 'rect'], 'Unknown annotation type.'

        self.root_dir = root_dir
        self.version = version
        self.anno_type = anno_type
        if download:
            self._download(root_dir, version)
        self.return_meta = return_meta

        if list_file is None:
            list_file = os.path.join(root_dir, 'list.txt')
        self._check_integrity(root_dir, version, list_file)

        with open(list_file, 'r') as f:
            self.seq_names = f.read().strip().split('\n')
        self.seq_dirs = [os.path.join(root_dir, s) for s in self.seq_names]
        self.anno_files = [os.path.join(s, 'groundtruth.txt')
                           for s in self.seq_dirs]

    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno) if ``return_meta`` is False, otherwise
                (img_files, anno, meta), where ``img_files`` is a list of
                file names, ``anno`` is a N x 4 (rectangles) or N x 8 (corners) numpy array,
                while ``meta`` is a dict contains meta information about the sequence.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        img_files = sorted(glob.glob(
            os.path.join(self.seq_dirs[index], 'color', '*.jpg')))
        anno = np.loadtxt(self.anno_files[index], delimiter=',')
        assert len(img_files) == len(anno), (len(img_files), len(anno))
        assert anno.shape[1] in [4, 8]
        if self.anno_type == 'rect' and anno.shape[1] == 8:
            anno = self._corner2rect(anno)

        if self.return_meta:
            meta = self._fetch_meta(
                self.seq_dirs[index], len(img_files))
            return img_files, anno, meta
        else:
            return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _download(self, root_dir, version):
        assert version in self.__valid_versions

        if not os.path.isdir(root_dir):
            os.makedirs(root_dir)
        elif os.path.isfile(os.path.join(root_dir, 'list.txt')):
            with open(os.path.join(root_dir, 'list.txt')) as f:
                seq_names = f.read().strip().split('\n')
            if all([os.path.isdir(os.path.join(root_dir, s)) for s in seq_names]):
                print('Files already downloaded.')
                return

        url = 'http://data.votchallenge.net/'
        if version in range(2013, 2015 + 1):
            # main challenge (2013~2015)
            homepage = url + 'vot{}/dataset/'.format(version)
        elif version in range(2015, 2019 + 1):
            # main challenge (2016~2019)
            homepage = url + 'vot{}/main/'.format(version)
        elif version.startswith('LT'):
            # long-term tracking challenge
            year = int(version[2:])
            homepage = url + 'vot{}/longterm/'.format(year)
        elif version.startswith('RGBD'):
            # RGBD tracking challenge
            year = int(version[4:])
            homepage = url + 'vot{}/rgbd/'.format(year)
        elif version.startswith('RGBT'):
            # RGBT tracking challenge
            year = int(version[4:])
            url = url + 'vot{}/rgbtir/'.format(year)
            homepage = url + 'meta/'
        
        # download description file
        bundle_url = homepage + 'description.json'
        bundle_file = os.path.join(root_dir, 'description.json')
        if not os.path.isfile(bundle_file):
            print('Downloading description file...')
            download(bundle_url, bundle_file)

        # read description file
        print('\nParsing description file...')
        with open(bundle_file) as f:
            bundle = json.load(f)

        # md5 generator
        def md5(filename):
            hash_md5 = hashlib.md5()
            with open(filename, 'rb') as f:
                for chunk in iter(lambda: f.read(4096), b""):
                    hash_md5.update(chunk)
            return hash_md5.hexdigest()
        
        # download all sequences
        seq_names = []
        for seq in bundle['sequences']:
            seq_name = seq['name']
            seq_names.append(seq_name)

            # download channel (color/depth/ir) files
            channels = seq['channels'].keys()
            seq_files = []
            for cn in channels:
                seq_url = seq['channels'][cn]['url']
                if not seq_url.startswith(('http', 'https')):
                    seq_url = url + seq_url[seq_url.find('sequence'):]
                seq_file = os.path.join(
                    root_dir,
                    '{}_{}.zip'.format(seq_name, cn))
                if not os.path.isfile(seq_file) or \
                    md5(seq_file) != seq['channels'][cn]['checksum']:
                    print('\nDownloading %s...' % seq_name)
                    download(seq_url, seq_file)
                seq_files.append(seq_file)

            # download annotations
            anno_url = homepage + '%s.zip' % seq_name
            anno_file = os.path.join(root_dir, seq_name + '_anno.zip')
            if not os.path.isfile(anno_file) or \
                md5(anno_file) != seq['annotations']['checksum']:
                download(anno_url, anno_file)

            # unzip compressed files
            seq_dir = os.path.join(root_dir, seq_name)
            if not os.path.isfile(seq_dir) or len(os.listdir(seq_dir)) < 10:
                print('Extracting %s...' % seq_name)
                os.makedirs(seq_dir)
                for seq_file in seq_files:
                    extract(seq_file, seq_dir)
                extract(anno_file, seq_dir)

        # save list.txt
        list_file = os.path.join(root_dir, 'list.txt')
        with open(list_file, 'w') as f:
            f.write(str.join('\n', seq_names))

        return root_dir

    def _check_integrity(self, root_dir, version, list_file=None):
        assert version in self.__valid_versions
        if list_file is None:
            list_file = os.path.join(root_dir, 'list.txt')

        if os.path.isfile(list_file):
            with open(list_file, 'r') as f:
                seq_names = f.read().strip().split('\n')

            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted. ' +
                            'You can use download=True to download it.')

    def _corner2rect(self, corners, center=False):
        cx = np.mean(corners[:, 0::2], axis=1)
        cy = np.mean(corners[:, 1::2], axis=1)

        x1 = np.min(corners[:, 0::2], axis=1)
        x2 = np.max(corners[:, 0::2], axis=1)
        y1 = np.min(corners[:, 1::2], axis=1)
        y2 = np.max(corners[:, 1::2], axis=1)

        area1 = np.linalg.norm(corners[:, 0:2] - corners[:, 2:4], axis=1) * \
            np.linalg.norm(corners[:, 2:4] - corners[:, 4:6], axis=1)
        area2 = (x2 - x1) * (y2 - y1)
        scale = np.sqrt(area1 / area2)
        w = scale * (x2 - x1) + 1
        h = scale * (y2 - y1) + 1

        if center:
            return np.array([cx, cy, w, h]).T
        else:
            return np.array([cx - w / 2, cy - h / 2, w, h]).T

    def _fetch_meta(self, seq_dir, frame_num):
        meta = {}

        # attributes
        tag_files = glob.glob(os.path.join(seq_dir, '*.label')) + \
            glob.glob(os.path.join(seq_dir, '*.tag'))
        for f in tag_files:
            tag = os.path.basename(f)
            tag = tag[:tag.rfind('.')]
            meta[tag] = np.loadtxt(f)
        
        # practical
        practical_file = os.path.join(seq_dir, 'practical')
        if os.path.isfile(practical_file + '.value'):
            meta['practical'] = np.loadtxt(practical_file + '.value')
        if os.path.isfile(practical_file + '.txt'):
            meta['practical_txt'] = np.loadtxt(practical_file + '.txt')

        # pad zeros if necessary
        for tag, val in meta.items():
            if len(val) < frame_num:
                meta[tag] = np.pad(
                    val, (0, frame_num - len(val)), 'constant')

        return meta
got10k/datasets/trackingnet.py
class TrackingNet(object):
    r"""`TrackingNet <https://tracking-net.org/>`_ Datasets.

    Publication:
        ``TrackingNet: A Large-Scale Dataset and Benchmark for Object Tracking in the Wild.``,
        M. Muller, A. Bibi, S. Giancola, S. Al-Subaihi and B. Ghanem, ECCV 2018.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
        subset (string, optional): Specify ``train`` or ``test``
            subset of TrackingNet.
    """
    def __init__(self, root_dir, subset='test', *args, **kwargs):
        super(TrackingNet, self).__init__()
        assert subset in ['train', 'test'], 'Unknown subset.'

        self.root_dir = root_dir
        self.subset = subset
        if subset == 'test':
            self.subset_dirs = ['TEST']
        elif subset == 'train':
            self.subset_dirs = ['TRAIN_%d' % c for c in range(12)]
        self._check_integrity(root_dir, self.subset_dirs)

        self.anno_files = [glob.glob(os.path.join(
            root_dir, c, 'anno/*.txt')) for c in self.subset_dirs]
        self.anno_files = sorted(sum(self.anno_files, []))
        self.seq_dirs = [os.path.join(
            os.path.dirname(os.path.dirname(f)),
            'frames',
            os.path.basename(f)[:-4])
            for f in self.anno_files]
        self.seq_names = [os.path.basename(d) for d in self.seq_dirs]

    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)
        
        img_files = glob.glob(
            os.path.join(self.seq_dirs[index], '*.jpg'))
        img_files = sorted(img_files, key=lambda x: int(os.path.basename(x)[:-4]))
        anno = np.loadtxt(self.anno_files[index], delimiter=',')
        # from IPython import embed;embed()
        if self.subset=='train':
            assert len(img_files) == len(anno)
            assert anno.shape[1] == 4
        elif self.subset=='test':
            assert anno.shape[0] == 4
        anno = anno.reshape(-1, 4)
        
        return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _check_integrity(self, root_dir, subset_dirs):
        # check each subset path
        for c in subset_dirs:
            subset_dir = os.path.join(root_dir, c)
            
            # check data and annotation folders
            for folder in ['anno', 'frames']:
                if not os.path.isdir(os.path.join(subset_dir, folder)):
                    raise Exception('Dataset not found or corrupted.')
got10k/datasets/dtb70.py
class DTB70(object):
    """`DTB70 <https://github.com/flyers/drone-tracking>`_ Dataset.

    Publication:
        ``Visual object tracking for unmanned aerial vehicles: A benchmark and new motion models``,
        Y. Wu, J. Lim and M.-H. Yang, IEEE TPAMI 2015.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
    """
    def __init__(self, root_dir):
        super(DTB70, self).__init__()
        self.root_dir = root_dir
        self._check_integrity(root_dir)

        self.anno_files = sorted(glob.glob(
            os.path.join(root_dir, '*/groundtruth_rect.txt')))
        self.seq_dirs = [os.path.dirname(f) for f in self.anno_files]
        self.seq_names = [os.path.basename(d) for d in self.seq_dirs]
    
    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        img_files = sorted(glob.glob(
            os.path.join(self.seq_dirs[index], 'img/*.jpg')))
        anno = np.loadtxt(self.anno_files[index], delimiter=',')
        assert len(img_files) == len(anno)
        assert anno.shape[1] == 4

        return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _check_integrity(self, root_dir):
        seq_names = os.listdir(root_dir)
        seq_names = [n for n in seq_names if not n[0] == '.']

        if os.path.isdir(root_dir) and len(seq_names) > 0:
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted.')
got10k/datasets/otb.py
class OTB(object):
    r"""`OTB <http://cvlab.hanyang.ac.kr/tracker_benchmark/>`_ Datasets.

    Publication:
        ``Object Tracking Benchmark``, Y. Wu, J. Lim and M.-H. Yang, IEEE TPAMI 2015.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
        version (integer or string): Specify the benchmark version, specify as one of
            ``2013``, ``2015``, ``tb50`` and ``tb100``.
        download (boolean, optional): If True, downloads the dataset from the internet
            and puts it in root directory. If dataset is downloaded, it is not
            downloaded again.
    """
    __otb13_seqs = ['Basketball', 'Bolt', 'Boy', 'Car4', 'CarDark',
                    'CarScale', 'Coke', 'Couple', 'Crossing', 'David',
                    'David2', 'David3', 'Deer', 'Dog1', 'Doll', 'Dudek',
                    'FaceOcc1', 'FaceOcc2', 'Fish', 'FleetFace',
                    'Football', 'Football1', 'Freeman1', 'Freeman3',
                    'Freeman4', 'Girl', 'Ironman', 'Jogging', 'Jumping',
                    'Lemming', 'Liquor', 'Matrix', 'Mhyang', 'MotorRolling',
                    'MountainBike', 'Shaking', 'Singer1', 'Singer2',
                    'Skating1', 'Skiing', 'Soccer', 'Subway', 'Suv',
                    'Sylvester', 'Tiger1', 'Tiger2', 'Trellis', 'Walking',
                    'Walking2', 'Woman']

    __tb50_seqs = ['Basketball', 'Biker', 'Bird1', 'BlurBody', 'BlurCar2',
                   'BlurFace', 'BlurOwl', 'Bolt', 'Box', 'Car1', 'Car4',
                   'CarDark', 'CarScale', 'ClifBar', 'Couple', 'Crowds',
                   'David', 'Deer', 'Diving', 'DragonBaby', 'Dudek',
                   'Football', 'Freeman4', 'Girl', 'Human3', 'Human4',
                   'Human6', 'Human9', 'Ironman', 'Jump', 'Jumping',
                   'Liquor', 'Matrix', 'MotorRolling', 'Panda', 'RedTeam',
                   'Shaking', 'Singer2', 'Skating1', 'Skating2', 'Skiing',
                   'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis',
                   'Walking', 'Walking2', 'Woman']

    __tb100_seqs = ['Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board',
                    'Bolt2', 'Boy', 'Car2', 'Car24', 'Coke', 'Coupon',
                    'Crossing', 'Dancer', 'Dancer2', 'David2', 'David3',
                    'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2', 'Fish',
                    'FleetFace', 'Football1', 'Freeman1', 'Freeman3',
                    'Girl2', 'Gym', 'Human2', 'Human5', 'Human7', 'Human8',
                    'Jogging', 'KiteSurf', 'Lemming', 'Man', 'Mhyang',
                    'MountainBike', 'Rubik', 'Singer1', 'Skater',
                    'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans',
                    'Twinnings', 'Vase'] + __tb50_seqs

    __otb15_seqs = __tb100_seqs

    __version_dict = {
        2013: __otb13_seqs,
        2015: __otb15_seqs,
        'otb2013': __otb13_seqs,
        'otb2015': __otb15_seqs,
        'tb50': __tb50_seqs,
        'tb100': __tb100_seqs}

    def __init__(self, root_dir, version=2015, download=True):
        super(OTB, self).__init__()
        assert version in self.__version_dict

        self.root_dir = root_dir
        self.version = version
        if download:
            self._download(root_dir, version)
        self._check_integrity(root_dir, version)

        valid_seqs = self.__version_dict[version]
        self.anno_files = sorted(list(chain.from_iterable(glob.glob(
            os.path.join(root_dir, s, 'groundtruth*.txt')) for s in valid_seqs)))
        # remove empty annotation files
        # (e.g., groundtruth_rect.1.txt of Human4)
        self.anno_files = self._filter_files(self.anno_files)
        self.seq_dirs = [os.path.dirname(f) for f in self.anno_files]
        self.seq_names = [os.path.basename(d) for d in self.seq_dirs]
        # rename repeated sequence names
        # (e.g., Jogging and Skating2)
        self.seq_names = self._rename_seqs(self.seq_names)

    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        img_files = sorted(glob.glob(
            os.path.join(self.seq_dirs[index], 'img/*.jpg')))

        # special sequences
        # (visit http://cvlab.hanyang.ac.kr/tracker_benchmark/index.html for detail)
        seq_name = self.seq_names[index]
        if seq_name.lower() == 'david':
            img_files = img_files[300-1:770]
        elif seq_name.lower() == 'football1':
            img_files = img_files[:74]
        elif seq_name.lower() == 'freeman3':
            img_files = img_files[:460]
        elif seq_name.lower() == 'freeman4':
            img_files = img_files[:283]
        elif seq_name.lower() == 'diving':
            img_files = img_files[:215]

        # to deal with different delimeters
        with open(self.anno_files[index], 'r') as f:
            anno = np.loadtxt(io.StringIO(f.read().replace(',', ' ')))
        assert len(img_files) == len(anno)
        assert anno.shape[1] == 4

        return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _filter_files(self, filenames):
        filtered_files = []
        for filename in filenames:
            with open(filename, 'r') as f:
                if f.read().strip() == '':
                    print('Warning: %s is empty.' % filename)
                else:
                    filtered_files.append(filename)

        return filtered_files

    def _rename_seqs(self, seq_names):
        # in case some sequences may have multiple targets
        renamed_seqs = []
        for i, seq_name in enumerate(seq_names):
            if seq_names.count(seq_name) == 1:
                renamed_seqs.append(seq_name)
            else:
                ind = seq_names[:i + 1].count(seq_name)
                renamed_seqs.append('%s.%d' % (seq_name, ind))

        return renamed_seqs

    def _download(self, root_dir, version):
        assert version in self.__version_dict
        seq_names = self.__version_dict[version]

        if not os.path.isdir(root_dir):
            os.makedirs(root_dir)
        elif all([os.path.isdir(os.path.join(root_dir, s)) for s in seq_names]):
            print('Files already downloaded.')
            return

        url_fmt = 'http://cvlab.hanyang.ac.kr/tracker_benchmark/seq/%s.zip'
        for seq_name in seq_names:
            seq_dir = os.path.join(root_dir, seq_name)
            if os.path.isdir(seq_dir):
                continue
            url = url_fmt % seq_name
            zip_file = os.path.join(root_dir, seq_name + '.zip')
            print('Downloading to %s...' % zip_file)
            download(url, zip_file)
            print('\nExtracting to %s...' % root_dir)
            extract(zip_file, root_dir)

        return root_dir

    def _check_integrity(self, root_dir, version):
        assert version in self.__version_dict
        seq_names = self.__version_dict[version]

        if os.path.isdir(root_dir) and len(os.listdir(root_dir)) > 0:
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(root_dir, seq_name)
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted. ' +
                            'You can use download=True to download it.')
got10k/datasets/uav123.py
class UAV123(object):
    """`UAV123 <https://ivul.kaust.edu.sa/Pages/pub-benchmark-simulator-uav.aspx>`_ Dataset.

    Publication:
        ``A Benchmark and Simulator for UAV Tracking``,
        M. Mueller, N. Smith and B. Ghanem, ECCV 2016.
    
    Args:
        root_dir (string): Root directory of dataset where sequence
            folders exist.
        version (integer or string): Specify the benchmark version, specify as one of
            ``UAV123`` and ``UAV20L``.
    """
    def __init__(self, root_dir, version='UAV123'):
        super(UAV123, self).__init__()
        assert version.upper() in ['UAV20L', 'UAV123']

        self.root_dir = root_dir
        self.version = version.upper()
        self._check_integrity(root_dir, version)

        # sequence meta information
        meta_file = os.path.join(
            os.path.dirname(__file__), 'uav123.json')
        with open(meta_file) as f:
            self.seq_metas = json.load(f)

        # sequence and annotation paths
        self.anno_files = sorted(glob.glob(
            os.path.join(root_dir, 'anno/%s/*.txt' % version)))
        self.seq_names = [
            os.path.basename(f)[:-4] for f in self.anno_files]
        self.seq_dirs = [os.path.join(
            root_dir, 'data_seq/UAV123/%s' % \
                self.seq_metas[version][n]['folder_name'])
            for n in self.seq_names]
    
    def __getitem__(self, index):
        r"""        
        Args:
            index (integer or string): Index or name of a sequence.
        
        Returns:
            tuple: (img_files, anno), where ``img_files`` is a list of
                file names and ``anno`` is a N x 4 (rectangles) numpy array.
        """
        if isinstance(index, six.string_types):
            if not index in self.seq_names:
                raise Exception('Sequence {} not found.'.format(index))
            index = self.seq_names.index(index)

        # valid frame range
        start_frame = self.seq_metas[self.version][
            self.seq_names[index]]['start_frame']
        end_frame = self.seq_metas[self.version][
            self.seq_names[index]]['end_frame']
        img_files = [os.path.join(
            self.seq_dirs[index], '%06d.jpg' % f)
            for f in range(start_frame, end_frame + 1)]

        # load annotations
        anno = np.loadtxt(self.anno_files[index], delimiter=',')
        assert len(img_files) == len(anno)
        assert anno.shape[1] == 4

        return img_files, anno

    def __len__(self):
        return len(self.seq_names)

    def _check_integrity(self, root_dir, version):
        # sequence meta information
        meta_file = os.path.join(
            os.path.dirname(__file__), 'uav123.json')
        with open(meta_file) as f:
            seq_metas = json.load(f)
        seq_names = list(seq_metas[version].keys())

        if os.path.isdir(root_dir) and len(os.listdir(root_dir)) > 3:
            # check each sequence folder
            for seq_name in seq_names:
                seq_dir = os.path.join(
                    root_dir, 'data_seq/UAV123/%s' % \
                        seq_metas[version][seq_name]['folder_name'])
                if not os.path.isdir(seq_dir):
                    print('Warning: sequence %s not exists.' % seq_name)
        else:
            # dataset not exists
            raise Exception('Dataset not found or corrupted.')


Question: import unittest
import os
import random
from got10k.datasets import GOT10k, OTB, VOT, DTB70, TColor128, \
    UAV123, NfS, LaSOT, TrackingNet, ImageNetVID
from __future__ import absolute_import




class TestDatasets(unittest.TestCase):

    def setUp(self):
        self.data_dir = os.path.expanduser('~/data')

    def tearDown(self):
        pass

    def test_got10k(self):
        root_dir = os.path.join(self.data_dir, 'GOT-10k')
        # without meta
        for subset in ['train', 'val', 'test']:
            dataset = GOT10k(root_dir, subset=subset)
            self._check_dataset(dataset)
        # with meta
        for subset in ['train', 'val', 'test']:
            dataset = GOT10k(root_dir, subset=subset, return_meta=True)
            self._check_dataset(dataset)

    def test_otb(self):
        root_dir = os.path.join(self.data_dir, 'OTB')
        dataset = OTB(root_dir)
        self._check_dataset(dataset)
    
    def test_vot(self):
        root_dir = os.path.join(self.data_dir, 'vot2018')
        # without meta
        dataset = VOT(root_dir, anno_type='rect')
        self._check_dataset(dataset)
        # with meta
        dataset = VOT(root_dir, anno_type='rect', return_meta=True)
        self._check_dataset(dataset)

    def test_dtb70(self):
        root_dir = os.path.join(self.data_dir, 'DTB70')
        dataset = DTB70(root_dir)
        self._check_dataset(dataset)
    
    def test_tcolor128(self):
        root_dir = os.path.join(self.data_dir, 'Temple-color-128')
        dataset = TColor128(root_dir)
        self._check_dataset(dataset)

    def test_uav123(self):
        root_dir = os.path.join(self.data_dir, 'UAV123')
        for version in ['UAV123', 'UAV20L']:

Answer:             dataset = UAV123(root_dir, version)