update on 2023-11-25

app.py

@@ -2,6 +2,8 @@ import os
 import json
 import math
 import base64
+from uuid import uuid4
+from PIL import Image as PILImage
 
 os.environ["DEEPFACE_HOME"] = "."
 
@@ -142,6 +144,60 @@ def face_distance_to_conf(face_distance, face_match_threshold=20.0):
         return linear_val + ((1.0 - linear_val) * math.pow((linear_val - 0.5) * 2, 0.2))
 
 
+def find_faces_in_sprite(image, vtt):
+    vtt = base64.b64decode(vtt.replace("data:text/vtt;base64,", ""))
+    sprite = PILImage.fromarray(image)
+
+    results = []
+    for i, (left, top, right, bottom, time_seconds) in enumerate(getVTToffsets(vtt)):
+        cut_frame = sprite.crop((left, top, left + right, top + bottom))
+        faces = DeepFace.extract_faces(np.asarray(cut_frame), detector_backend="mediapipe", enforce_detection=False, align=False)
+        faces = [face for face in faces if face['confidence'] > 0.75]
+        if faces:
+            size = faces[0]['facial_area']['w'] * faces[0]['facial_area']['h']
+            data = {'id': str(uuid4()), "offset": (left, top, right, bottom), "frame": i, "time": time_seconds, 'size': size}
+            results.append(data)
+
+    # sort by size
+    results = sorted(results, key=lambda x: x['size'], reverse=True)
+    return results
+
+
+def getVTToffsets(vtt):
+    time_seconds = 0
+    left = top = right = bottom = None
+    for line in vtt.decode("utf-8").split("\n"):
+        line = line.strip()
+
+        if "-->" in line:
+            # grab the start time
+            # 00:00:00.000 --> 00:00:41.000
+            start = line.split("-->")[0].strip().split(":")
+            # convert to seconds
+            time_seconds = (
+                int(start[0]) * 3600
+                + int(start[1]) * 60
+                + float(start[2])
+            )
+            left = top = right = bottom = None
+        elif "xywh=" in line:
+            left, top, right, bottom = line.split("xywh=")[-1].split(",")
+            left, top, right, bottom = (
+                int(left),
+                int(top),
+                int(right),
+                int(bottom),
+            )
+        else:
+            continue
+
+        if not left:
+            continue
+
+        yield left, top, right, bottom, time_seconds
+
+
+
 image_search = gr.Interface(
     fn=image_search_performer,
     inputs=[
@@ -178,4 +234,13 @@ vector_search = gr.Interface(
     description="512 vector created with deepface of a person and we'll tell you who it is.",
 )
 
-gr.TabbedInterface([image_search, image_search_multiple, vector_search]).launch(enable_queue=True, server_name="0.0.0.0")
+faces_in_sprite = gr.Interface(
+    fn=find_faces_in_sprite,
+    inputs=[
+        gr.Image(),
+        gr.Textbox(label="VTT file")
+    ],
+    outputs=gr.JSON(label=""),
+)
+
+gr.TabbedInterface([image_search, image_search_multiple, vector_search, faces_in_sprite]).launch(enable_queue=True, server_name="0.0.0.0")

face.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa2e948f81e03173f911f677315110792c5aa186c4d8ab6b00b84c7aedcf1541
-size 648537696
+oid sha256:83ddc89e7e2f7f9db6eccd759f2dbcafa9527fd8f48c2e1f0c513cb7efe3fa0a
+size 653004192

face.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8042e9467c7ca01a3daeca9e7558976c957324c0bd39e9dbb8b02714e29551e
-size 8123440
+oid sha256:0872787307365930fef6283cb45ab9883a09e56ed39e0403182831472b73eebe
+size 8180960

persons.zip

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad2dc27be78018ad91f3c71a18e80728f634858cbb2383f06c651f084dc5a134
-size 3864732
+oid sha256:fa6d657dff6f902d4ecd9e004ee81dea78bd0d1720de30b4218f93b58f5eb85c
+size 3885315

requirements.txt

@@ -103,4 +103,5 @@ websockets==10.4
 Werkzeug==2.2.3
 wrapt==1.15.0
 yarl==1.8.2
+mediapipe==0.10.8
 gradio