new

app.py

@@ -1,4 +1,3 @@
-# Import the required libraries
 import os
 
 import cv2
@@ -9,7 +8,13 @@ import tensorflow as tf
 import tensorflow.lite as tflite
 
 # Initialize MediaPipe solutions
-mp_holistic = mp.solutions.holistic
+mp_hands = mp.solutions.hands
+mp_pose = mp.solutions.pose
+mp_face_mesh = mp.solutions.face_mesh
+
+hands = mp_hands.Hands()
+pose = mp_pose.Pose()
+face_mesh = mp_face_mesh.FaceMesh()
 
 # Get the absolute path to the directory containing app.py
 current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -17,56 +22,61 @@ current_dir = os.path.dirname(os.path.abspath(__file__))
 model_filename = "model.tflite"
 # Construct the full path to the TFLite model file
 model_path = os.path.join(current_dir, model_filename)
-
 # Load the TFLite model using the interpreter
 interpreter = tf.lite.Interpreter(model_path=model_path)
 interpreter.allocate_tensors()
 
 
-# ... (other functions from previous code)
+# Preprocess landmarks
+def preprocess_landmarks(hand1_landmarks, hand2_landmarks, pose_landmarks, lip_landmarks):
+    hand1_landmarks = [[landmark.x, landmark.y, landmark.z] for landmark in hand1_landmarks.landmark]
+    hand2_landmarks = [[landmark.x, landmark.y, landmark.z] for landmark in hand2_landmarks.landmark]
+    pose_landmarks = [[landmark.x, landmark.y, landmark.z] for landmark in pose_landmarks.landmark]
+    lip_landmarks = [[landmark.x, landmark.y, landmark.z] for landmark in lip_landmarks]
+
+    combined_landmarks = lip_landmarks + hand1_landmarks + hand2_landmarks + pose_landmarks
+
+    return np.array(combined_landmarks, dtype=np.float32)
+
+
+# Function to extract landmarks from the webcam frame
+def extract_landmarks(frame):
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    results = hands.process(frame_rgb)
+    pose_results = pose.process(frame_rgb)
+    face_results = face_mesh.process(frame_rgb)
+
+    if not results.multi_hand_landmarks or not pose_results.pose_landmarks or not face_results.multi_face_landmarks:
+        return None
+
+    hand1_landmarks = results.multi_hand_landmarks[0]
+    if len(results.multi_hand_landmarks) > 1:
+        hand2_landmarks = results.multi_hand_landmarks[1]
+    else:
+        hand2_landmarks = hand1_landmarks
 
-# Function to perform holistic detection using Mediapipe
-def mediapipe_detection(image, model):
-    # COLOR CONVERSION BGR 2 RGB
-    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-    image_rgb.flags.writeable = False  # Image is no longer writeable
-    results = model.process(image_rgb)  # Make prediction
-    image_rgb.flags.writeable = True  # Image is now writeable
-    image_bgr = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)  # COLOR CONVERSION RGB 2 BGR
-    return image_bgr, results
+    pose_landmarks = pose_results.pose_landmarks
+    face_landmarks = face_results.multi_face_landmarks[0]
+    lip_landmarks = [face_landmarks.landmark[i] for i in LIPS_IDXS0 - START_IDX]
 
+    return hand1_landmarks, hand2_landmarks, pose_landmarks, lip_landmarks
 
-# Function to extract keypoints from Mediapipe results
-# Function to extract keypoints from Mediapipe results
-def extract_keypoints(results):
-    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten(
-    ) if results.left_hand_landmarks else np.zeros(21 * 3)
-    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten(
-    ) if results.right_hand_landmarks else np.zeros(21 * 3)
-    pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten(
-    ) if results.pose_landmarks else np.zeros(33 * 4)
-    face = np.array([[res.x, res.y, res.z] for res in
-                     results.face_landmarks.landmark]).flatten(
-    ) if results.face_landmarks else np.zeros(468 * 3)
 
-    # Concatenate the arrays and return the result
-    keypoints = np.concatenate([lh, rh, pose, face])
-    return keypoints
+# Make prediction
+def make_prediction(processed_landmarks):
+    inputs = np.array([processed_landmarks])
+    interpreter.set_tensor(input_details[0]['index'], inputs)
+    interpreter.invoke()
+    outputs = interpreter.get_tensor(output_details[0]['index'])
+    return outputs[0].argmax()
 
 
-# Main prediction function that combines everything
+# Gradio Interface Function
 def predict_with_webcam(frame):
-    # Perform holistic detection
-    image, results = mediapipe_detection(frame, mp_holistic)  # Use mp_holistic here
-    # Extract keypoints
-    keypoints = extract_keypoints(results)
-    if np.count_nonzero(keypoints) > 0:
-        # Preprocess keypoints and make prediction
-        processed_landmarks = np.array([keypoints], dtype=np.float32)
-        interpreter.set_tensor(input_details[0]['index'], processed_landmarks)
-        interpreter.invoke()
-        outputs = interpreter.get_tensor(output_details[0]['index'])
-        prediction = outputs[0].argmax()
+    landmarks = extract_landmarks(frame)
+    if landmarks is not None:
+        processed_landmarks = preprocess_landmarks(*landmarks)
+        prediction = make_prediction(processed_landmarks)
         return str(prediction)
 
 
@@ -83,4 +93,4 @@ webcam_interface = gr.Interface(
 
 # Launch the Gradio app with the webcam interface
 if __name__ == "__main__":
-    webcam_interface.launch()
+    webcam_interface.launch(share=True)