def unzip_fn(items: list): return [list(i) for i in zip(*items)] def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False): ''' Remove duplicates from a list of items Args: items: List of items key: Function to get the key of the item show_process: Whether to show the process or not Returns: List: List of items without duplicates ''' progress = lambda x, *, desc: x if show_process: import tqdm progress = tqdm.tqdm deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values()) return deduped_items if not unzip else unzip_fn(deduped_items)