File size: 686 Bytes
9c6eceb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
def unzip_fn(items: list):
return [list(i) for i in zip(*items)]
def remove_duplicates(items: list, key=lambda x: x, show_process=False, unzip=False):
'''
Remove duplicates from a list of items
Args:
items: List of items
key: Function to get the key of the item
show_process: Whether to show the process or not
Returns:
List: List of items without duplicates
'''
progress = lambda x, *, desc: x
if show_process:
import tqdm
progress = tqdm.tqdm
deduped_items = list({key(item): item for item in progress(items, desc='Deduping...')}.values())
return deduped_items if not unzip else unzip_fn(deduped_items) |