| def unzip_fn(items: list): | |
| return [list(i) for i in zip(*items)] | |
| def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False): | |
| ''' | |
| Remove duplicates from a list of items | |
| Args: | |
| items: List of items | |
| key: Function to get the key of the item | |
| show_process: Whether to show the process or not | |
| Returns: | |
| List: List of items without duplicates | |
| ''' | |
| progress = lambda x, *, desc: x | |
| if show_process: | |
| import tqdm | |
| progress = tqdm.tqdm | |
| deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values()) | |
| return deduped_items if not unzip else unzip_fn(deduped_items) |