utils/remove_duplicates.py

def unzip_fn(items: list):
    return [list(i) for i in zip(*items)]

def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
    '''
    Remove duplicates from a list of items
    Args:
      items: List of items
      key: Function to get the key of the item
      show_process: Whether to show the process or not
    Returns:
      List: List of items without duplicates
    '''
    progress = lambda x, *, desc: x
    if show_process:
        import tqdm
        progress = tqdm.tqdm
    deduped_items =  list({key(item): item for item in progress(items, desc='Deduping...')}.values())
    return deduped_items if not unzip else unzip_fn(deduped_items)