Update off_topic.py

off_topic.py

@@ -9,6 +9,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 import torch
 import PIL
+import imagehash
 from transformers import CLIPModel, CLIPProcessor
 from PIL import Image
 
@@ -95,7 +96,12 @@ class OffTopicDetector:
         domain = re.sub("_", " ", response["domain_id"].split("-")[-1]).lower()
         img_urls = [x["url"] for x in response["pictures"]]
         images = self.get_images(img_urls)
-        return images, domain
+        hashes = {}
+        for img in images:
+            hashes.update({str(imagehash.average_hash(img)): img})
+        dedup_hashes = list(dict.fromkeys(hashes))
+        dedup_images = [img for hash, img in hashes.items() if hash in dedup_hashes]
+        return dedup_images, domain
 
     def get_images(self, urls: List[str]):
         start = time.time()