create prototype for is someone is there

charles_actor.py

@@ -44,6 +44,11 @@ class CharlesActor:
             # "hello, how are you today?",
             # "hmm, interesting, tell me more about that.",
         ]
+
+        print("004")
+        print("creating prototypes")
+        from prototypes import Prototypes
+        self._prototypes = Prototypes()
         print("010")
         self._needs_init = True
         self._state = "Initialized"
@@ -99,8 +104,12 @@ class CharlesActor:
                 total_video_frames += 1
                 skipped_video_frames += (len(video_frames) -1)
                 image_as_array = video_frames[-1]
-                last_frame_vector = self._clip_transform.image_to_embeddings(image_as_array)
-                vector_debug = f"Last frame vector: {last_frame_vector.shape}"
+                image_vector = self._clip_transform.image_to_embeddings(image_as_array)
+                # image_vector = image_vector.unsqueeze(0)
+                image_vector = image_vector[0]
+                print(f"image_vector.shape: {image_vector.shape}")
+                distances, closest_item_key, distance_debug_str = self._prototypes.get_distances(image_vector)
+                vector_debug = f"{closest_item_key} {distance_debug_str}"
             
             await asyncio.sleep(0.01)
             loops+=1

clip_transform.py

@@ -48,6 +48,13 @@ class CLIPTransform:
             image_embeddings = self.model.encode_image(prepro)
         image_embeddings /= image_embeddings.norm(dim=-1, keepdim=True)
         return(image_embeddings)
+    
+    def pil_image_to_embeddings(self, input_im):
+        prepro = self.preprocess(input_im).unsqueeze(0).to(self.device)
+        with torch.no_grad():
+            image_embeddings = self.model.encode_image(prepro)
+        image_embeddings /= image_embeddings.norm(dim=-1, keepdim=True)
+        return(image_embeddings)
 
     def preprocessed_image_to_emdeddings(self, prepro):
         with torch.no_grad():

prototypes.py

@@ -0,0 +1,71 @@
+
+import os
+
+import torch
+from clip_transform import CLIPTransform
+from PIL import Image
+
+from torch.nn import functional as F
+
+class Prototypes:
+    def __init__(self):
+        self._clip_transform = CLIPTransform()
+        self._prepare_prototypes()
+
+
+    def _prepare_prototypes(self):
+        image_embeddings = self.load_images_from_folder('prototypes')
+        assert image_embeddings is not None, "no image embeddings found"
+        assert len(image_embeddings) > 0, "no image embeddings found"
+        person_keys = [key for key in image_embeddings.keys() if key.startswith('person-')]
+        no_person_keys = [key for key in image_embeddings.keys() if key.startswith('no_person-')]
+        person_keys.sort()
+        no_person_keys.sort()
+        # create pytorch vector of person embeddings
+        person_embeddings = torch.cat([image_embeddings[key] for key in person_keys])
+        # create pytorch vector of no_person embeddings
+        no_person_embeddings = torch.cat([image_embeddings[key] for key in no_person_keys])
+        person_embedding = person_embeddings.mean(dim=0)
+        person_embedding /= person_embedding.norm(dim=-1, keepdim=True)
+        no_person_embedding = no_person_embeddings.mean(dim=0)
+        no_person_embedding /= no_person_embedding.norm(dim=-1, keepdim=True)
+
+        self.prototype_keys = ["person", "no_person"]
+        self.prototypes = torch.stack([person_embedding, no_person_embedding])
+
+
+    def load_images_from_folder(self, folder):
+        image_embeddings = {}
+        supported_filetypes = ['.jpg','.png','.jpeg']
+        for filename in os.listdir(folder):
+            if not any([filename.endswith(ft) for ft in supported_filetypes]):
+                continue
+            image = Image.open(os.path.join(folder,filename))
+            embeddings = self._clip_transform.pil_image_to_embeddings(image)
+            image_embeddings[filename] = embeddings
+        return image_embeddings
+    
+    def get_distances(self, embeddings):
+        # case not normalized
+        # distances = F.cosine_similarity(embeddings, self.prototypes)
+        # case normalized
+        distances = embeddings @ self.prototypes.T
+        closest_item_idex = distances.argmax().item()
+        closest_item_key = self.prototype_keys[closest_item_idex]
+        debug_str = ""
+        for key, value in zip(self.prototype_keys, distances):
+            debug_str += f"{key}: {value.item():.2f}, "
+        return distances, closest_item_key, debug_str
+    
+
+if __name__ == "__main__":
+    prototypes = Prototypes()
+    print ("prototypes:")
+    for key, value in zip(prototypes.prototype_keys, prototypes.prototypes):
+        print (f"{key}: {len(value)}")
+
+    embeddings = prototypes.prototypes[0]
+    distances, closest_item_key, debug_str = prototypes.get_distances(embeddings)
+    print (f"closest_item_key: {closest_item_key}")
+    print (f"distances: {debug_str}")
+    print ("done")