app.py

"""
Facial Expression Recognition App
LittleMonkeyLab | Goldsmiths Observatory
"""

import gradio as gr
import cv2
import mediapipe as mp
import numpy as np
import os
from datetime import datetime

# Initialize MediaPipe Face Mesh
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(
    static_image_mode=True,
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5
)

# Define key facial landmarks for expressions
FACIAL_LANDMARKS = {
    'left_brow': [52, 65, 46],  # inner, middle, outer
    'right_brow': [285, 295, 276],  # inner, middle, outer
    'left_eye': [159, 145, 133],  # top, bottom, outer
    'right_eye': [386, 374, 362],  # top, bottom, outer
    'nose': [6, 197],  # bridge, tip
    'mouth': [61, 291, 0, 17, 13, 14],  # left corner, right corner, top lip, bottom lip, upper inner, lower inner
    'jaw': [17, 84, 314]  # center, left, right
}

def calculate_distances(points, landmarks):
    """Calculate normalized distances between facial landmarks."""
    def distance(p1_idx, p2_idx):
        try:
            p1 = points[p1_idx]
            p2 = points[p2_idx]
            return np.linalg.norm(p1 - p2)
        except:
            return 0.0
    
    # Get face height for normalization
    face_height = distance(FACIAL_LANDMARKS['nose'][0], FACIAL_LANDMARKS['jaw'][0])
    if face_height == 0:
        return {}
    
    measurements = {
        # Inner brow raising (AU1)
        'inner_brow_raise': (
            distance(FACIAL_LANDMARKS['left_brow'][0], FACIAL_LANDMARKS['nose'][0]) +
            distance(FACIAL_LANDMARKS['right_brow'][0], FACIAL_LANDMARKS['nose'][0])
        ) / (2 * face_height),
        
        # Outer brow raising (AU2)
        'outer_brow_raise': (
            distance(FACIAL_LANDMARKS['left_brow'][2], FACIAL_LANDMARKS['nose'][0]) +
            distance(FACIAL_LANDMARKS['right_brow'][2], FACIAL_LANDMARKS['nose'][0])
        ) / (2 * face_height),
        
        # Brow lowering (AU4)
        'brow_furrow': distance(FACIAL_LANDMARKS['left_brow'][0], FACIAL_LANDMARKS['right_brow'][0]) / face_height,
        
        # Eye opening (AU5)
        'eye_opening': (
            distance(FACIAL_LANDMARKS['left_eye'][0], FACIAL_LANDMARKS['left_eye'][1]) +
            distance(FACIAL_LANDMARKS['right_eye'][0], FACIAL_LANDMARKS['right_eye'][1])
        ) / (2 * face_height),
        
        # Smile width (AU12)
        'smile_width': distance(FACIAL_LANDMARKS['mouth'][0], FACIAL_LANDMARKS['mouth'][1]) / face_height,
        
        # Mouth height (AU25/26)
        'mouth_opening': distance(FACIAL_LANDMARKS['mouth'][4], FACIAL_LANDMARKS['mouth'][5]) / face_height,
        
        # Lip corner height (for smile/frown detection)
        'lip_corner_height': (
            (points[FACIAL_LANDMARKS['mouth'][0]][1] + points[FACIAL_LANDMARKS['mouth'][1]][1])/2 -
            points[FACIAL_LANDMARKS['mouth'][2]][1]
        ) / face_height
    }
    
    return measurements

def analyze_expression(image):
    if image is None:
        return None, "No image provided", None
    
    # Convert to RGB if needed
    if len(image.shape) == 2:
        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
    elif image.shape[2] == 4:
        image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB)
    
    # Process the image
    results = face_mesh.process(image)
    
    if not results.multi_face_landmarks:
        return None, "No face detected", None
    
    # Get landmarks
    landmarks = results.multi_face_landmarks[0]
    points = np.array([[lm.x, lm.y, lm.z] for lm in landmarks.landmark])
    
    # Calculate facial measurements
    measurements = calculate_distances(points, landmarks)
    
    # Analyze Action Units with refined thresholds
    aus = {
        'AU01': measurements['inner_brow_raise'] > 0.12,  # Inner Brow Raiser
        'AU02': measurements['outer_brow_raise'] > 0.12,  # Outer Brow Raiser
        'AU04': measurements['brow_furrow'] < 0.2,        # Brow Lowerer (tighter threshold for anger)
        'AU05': measurements['eye_opening'] > 0.1,        # Upper Lid Raiser
        'AU12': measurements['smile_width'] > 0.45,       # Lip Corner Puller
        'AU25': measurements['mouth_opening'] > 0.08,     # Lips Part
        'AU26': measurements['mouth_opening'] > 0.15      # Jaw Drop
    }
    
    # Refined emotion classification with mutual exclusion
    emotions = {}
    
    # Check Anger first (takes precedence due to distinctive features)
    if aus['AU04'] and not aus['AU12']:  # Lowered brows without smile
        emotions["Angry"] = True
    # Happy - clear smile without anger indicators
    elif aus['AU12'] and measurements['lip_corner_height'] < -0.02 and not aus['AU04']:
        emotions["Happy"] = True
    # Sad - raised inner brow with neutral/down mouth
    elif aus['AU01'] and measurements['lip_corner_height'] > 0.01 and not aus['AU12']:
        emotions["Sad"] = True
    # Surprised - raised brows with open mouth
    elif (aus['AU01'] or aus['AU02']) and (aus['AU25'] or aus['AU26']) and not aus['AU04']:
        emotions["Surprised"] = True
    # Neutral - no strong indicators of other emotions
    elif not any([aus['AU01'], aus['AU02'], aus['AU04'], aus['AU12'], aus['AU26']]) and abs(measurements['lip_corner_height']) < 0.02:
        emotions["Neutral"] = True
    else:
        emotions["Neutral"] = True  # Default to neutral if no clear emotion is detected
    
    # Create visualization
    viz_image = image.copy()
    h, w = viz_image.shape[:2]
    
    # Draw facial landmarks with different colors for key points
    colors = {
        'brow': (0, 255, 0),    # Green
        'eye': (255, 255, 0),   # Yellow
        'nose': (0, 255, 255),  # Cyan
        'mouth': (255, 0, 255), # Magenta
        'jaw': (255, 128, 0)    # Orange
    }
    
    # Draw landmarks with feature-specific colors - made more visible
    for feature, points_list in FACIAL_LANDMARKS.items():
        color = colors.get(feature.split('_')[0], (0, 255, 0))
        for point_idx in points_list:
            pos = (int(landmarks.landmark[point_idx].x * w), 
                  int(landmarks.landmark[point_idx].y * h))
            # Larger circles with white outline for visibility
            cv2.circle(viz_image, pos, 4, (255, 255, 255), -1)  # White background
            cv2.circle(viz_image, pos, 3, color, -1)  # Colored center
    
    # Add emotion text
    detected_emotions = [emotion for emotion, is_present in emotions.items() if is_present]
    emotion_text = " + ".join(detected_emotions) if detected_emotions else "Neutral"
    
    # Create detailed analysis text
    analysis = f"Expression: {emotion_text}\n\nActive Action Units:\n"
    au_descriptions = {
        'AU01': 'Inner Brow Raiser',
        'AU02': 'Outer Brow Raiser',
        'AU04': 'Brow Lowerer',
        'AU05': 'Upper Lid Raiser',
        'AU12': 'Lip Corner Puller (Smile)',
        'AU25': 'Lips Part',
        'AU26': 'Jaw Drop'
    }
    
    active_aus = [f"{au}" for au, active in aus.items() if active]
    aus_text = "_".join(active_aus) if active_aus else "NoAUs"
    
    # Create filename with timestamp, emotion, and AUs
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    download_filename = f"FER_{timestamp}_{emotion_text.replace(' + ', '_')}_{aus_text}.jpg"
    
    # Add text with black background
    font = cv2.FONT_HERSHEY_SIMPLEX
    font_scale = 0.7
    thickness = 2
    y_pos = 30
    
    for line in emotion_text.split('\n'):
        (text_w, text_h), _ = cv2.getTextSize(line, font, font_scale, thickness)
        cv2.rectangle(viz_image, (10, y_pos - text_h - 5), (text_w + 20, y_pos + 5), (0, 0, 0), -1)
        cv2.putText(viz_image, line, (15, y_pos), font, font_scale, (255, 255, 255), thickness)
        y_pos += text_h + 20
    
    return viz_image, analysis, download_filename

def save_original_image(image, filename):
    if image is None or filename is None:
        return None
    return image

# Create Gradio interface
with gr.Blocks(css="app.css") as demo:
    # Header with Observatory logo
    with gr.Row(elem_classes="header-container"):
        with gr.Column():
            gr.Image("images/LMLOBS.png", show_label=False, container=False, elem_classes="header-logo")
    
    gr.Markdown("# Facial Expression Recognition")
    gr.Markdown("### LittleMonkeyLab | Goldsmiths Observatory")
    
    with gr.Row():
        with gr.Column():
            input_image = gr.Image(label="Upload Image", type="numpy")
            download_button = gr.Button("Download Original Image with Expression", visible=False)
            gr.Markdown("""
            ### Instructions:
            1. Upload a clear facial image
            2. View the detected expression and Action Units (AUs)
            3. Colored dots show key facial features:
               - Green: Eyebrows
               - Yellow: Eyes
               - Cyan: Nose
               - Magenta: Mouth
               - Orange: Jaw
            4. Click 'Download' to save the original image
            """)
        
        with gr.Column():
            output_image = gr.Image(label="Analysis")
            analysis_text = gr.Textbox(label="Expression Analysis", lines=8)
            download_output = gr.File(label="Download", visible=False)
    
    # Footer
    with gr.Row(elem_classes="center-content"):
        with gr.Column():
            gr.Image("images/LMLLOGO.png", show_label=False, container=False, elem_classes="footer-logo")
            gr.Markdown("© LittleMonkeyLab | Goldsmiths Observatory", elem_classes="footer-text")
    
    # Set up the event handlers
    filename = gr.State()
    
    def update_interface(image):
        viz_image, analysis, download_name = analyze_expression(image)
        download_button.visible = True if image is not None else False
        return viz_image, analysis, download_name
    
    input_image.change(
        fn=update_interface,
        inputs=[input_image],
        outputs=[output_image, analysis_text, filename]
    )
    
    download_button.click(
        fn=save_original_image,
        inputs=[input_image, filename],
        outputs=download_output
    )

if __name__ == "__main__":
    demo.launch()