upgrades deduplicateAndSort