Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import mediapipe as mp | |
| import numpy as np | |
| import log_utils | |
| from functools import lru_cache | |
| import cv2 | |
| from google.protobuf.json_format import MessageToDict | |
| logger = log_utils.get_logger() | |
| mp_hands = mp.solutions.hands | |
| mp_hands_connections = mp.solutions.hands_connections | |
| mp_draw = mp.solutions.drawing_utils | |
| connections = { | |
| 'HAND_CONNECTIONS': mp_hands_connections.HAND_CONNECTIONS, | |
| 'HAND_PALM_CONNECTIONS': mp_hands_connections.HAND_PALM_CONNECTIONS, | |
| 'HAND_THUMB_CONNECTIONS': mp_hands_connections.HAND_THUMB_CONNECTIONS, | |
| 'HAND_INDEX_FINGER_CONNECTIONS': mp_hands_connections.HAND_INDEX_FINGER_CONNECTIONS, | |
| 'HAND_MIDDLE_FINGER_CONNECTIONS': mp_hands_connections.HAND_MIDDLE_FINGER_CONNECTIONS, | |
| 'HAND_RING_FINGER_CONNECTIONS': mp_hands_connections.HAND_RING_FINGER_CONNECTIONS, | |
| 'HAND_PINKY_FINGER_CONNECTIONS': mp_hands_connections.HAND_PINKY_FINGER_CONNECTIONS, | |
| } | |
| def get_model(static_image_mode, max_num_hands, model_complexity, min_detection_conf, min_tracking_conf): | |
| return mp_hands.Hands( | |
| static_image_mode=static_image_mode, | |
| max_num_hands=max_num_hands, | |
| model_complexity=model_complexity, | |
| min_detection_confidence=min_detection_conf, | |
| min_tracking_confidence=min_tracking_conf, | |
| ) | |
| def draw_landmarks(model, img, selected_connections, draw_background, flip_image): | |
| img_to_process = cv2.flip(img, 1) if flip_image else img | |
| results = model.process(img_to_process) | |
| output_img = img_to_process if draw_background else np.zeros_like(img_to_process) | |
| if results.multi_hand_landmarks: | |
| for hand_landmarks in results.multi_hand_landmarks: | |
| mp_draw.draw_landmarks(output_img, hand_landmarks, connections[selected_connections]) | |
| if flip_image: | |
| output_img = cv2.flip(output_img, 1) | |
| return output_img, [MessageToDict(h) for _, h in enumerate(results.multi_handedness or [])] | |
| def process_image( | |
| img, | |
| static_image_mode, | |
| max_num_hands, | |
| model_complexity, | |
| min_detection_conf, | |
| min_tracking_conf, | |
| selected_connections, | |
| draw_background, | |
| flip_image, | |
| ): | |
| logger.info(f"Processing image with connections: {selected_connections}, draw background: {draw_background}") | |
| model = get_model(static_image_mode, max_num_hands, model_complexity, min_detection_conf, min_tracking_conf) | |
| img, multi_handedness = draw_landmarks(model, img, selected_connections, draw_background, flip_image) | |
| left_hand_count = len([h for h in multi_handedness if h['classification'][0]['label'] == 'Left']) | |
| right_hand_count = len(multi_handedness) - left_hand_count | |
| return img, multi_handedness, left_hand_count, right_hand_count | |
| demo = gr.Blocks() | |
| with demo: | |
| gr.Markdown( | |
| """ | |
| # MediaPipe's Hand & Finger Tracking | |
| A demo of hand and finger tracking using [Google's MediaPipe](https://google.github.io/mediapipe/solutions/hands.html). | |
| """) | |
| with gr.Column(): | |
| gr.Markdown(""" | |
| ## Step 1: Configure the model | |
| """) | |
| with gr.Column(): | |
| static_image_mode = gr.Checkbox(label="Static image mode", value=False) | |
| gr.Textbox(show_label=False,value="If unchecked, the solution treats the input images as a video stream. It will try to detect hands in the first input images, and upon a successful detection further localizes the hand landmarks. In subsequent images, once all max_num_hands hands are detected and the corresponding hand landmarks are localized, it simply tracks those landmarks without invoking another detection until it loses track of any of the hands. This reduces latency and is ideal for processing video frames. If checked, hand detection runs on every input image, ideal for processing a batch of static, possibly unrelated, images.") | |
| max_num_hands = gr.Slider(label="Max number of hands to detect", value=1, minimum=1, maximum=10, step=1) | |
| with gr.Column(): | |
| model_complexity = gr.Radio(label="Model complexity", choices=[0,1], value=1) | |
| gr.Textbox(show_label=False, value="Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as inference latency generally go up with the model complexity.") | |
| with gr.Column(): | |
| min_detection_conf = gr.Slider(label="Min detection confidence", value=0.5, minimum=0.0, maximum=1.0, step=0.1) | |
| gr.Textbox(show_label=False, value="Minimum confidence value ([0.0, 1.0]) from the hand detection model for the detection to be considered successful.") | |
| with gr.Column(): | |
| min_tracking_conf = gr.Slider(label="Min tracking confidence", value=0.5, minimum=0.0, maximum=1.0, step=0.1) | |
| gr.Textbox(show_label=False, value="Minimum confidence value ([0.0, 1.0]) from the landmark-tracking model for the hand landmarks to be considered tracked successfully, or otherwise hand detection will be invoked automatically on the next input image. Setting it to a higher value can increase robustness of the solution, at the expense of a higher latency. Ignored if static_image_mode is true, where hand detection simply runs on every image.") | |
| gr.Markdown(""" | |
| ## Step 2: Set processing parameters | |
| """) | |
| draw_background = gr.Checkbox(value=True, label="Draw background?") | |
| flip_image = gr.Checkbox(value=True, label="Flip image? (Note that handedness is determined assuming the input image is mirrored, i.e., taken with a front-facing/selfie camera with images flipped horizontally. If it is not the case, please swap the handedness output in the application.)") | |
| connection_keys = list(connections.keys()) | |
| selected_connections = gr.Dropdown( | |
| label="Select connections to draw", | |
| choices=connection_keys, | |
| value=connection_keys[0], | |
| ) | |
| gr.Markdown(""" | |
| ## Step 3: Select an image | |
| """) | |
| with gr.Tabs(): | |
| with gr.TabItem(label="Upload an image"): | |
| uploaded_image = gr.Image(type="numpy", label="Input image") | |
| example_image = gr.Examples(examples=[['examples/example-01.jpg', 1, 0.4], ['examples/example-02.jpg', 2, 0.5], ['examples/example-03.jpg', 1, 0.5]], inputs=[uploaded_image, max_num_hands, min_detection_conf]) | |
| submit_uploaded_image = gr.Button(value="Process Image") | |
| with gr.TabItem(label="Take a picture"): | |
| camera_picture = gr.Image(source="webcam", type="numpy", label="Input image") | |
| submit_camera_picture = gr.Button(value="Process Image") | |
| gr.Markdown(""" | |
| ## Step 4: View results | |
| """) | |
| with gr.Column(): | |
| with gr.Row(): | |
| with gr.Column(): | |
| left_hands = gr.Number(label="Left hands detected") | |
| with gr.Column(): | |
| right_hands = gr.Number(label="Right hands detected") | |
| multi_handedness = gr.JSON(label="Raw handedness results") | |
| processed_image = gr.Image(label="Processed image") | |
| gr.Markdown('<img id="visitor-badge" alt="visitor badge" src="https://visitor-badge.glitch.me/badge?page_id=kristyc.mediapipe-hands" />') | |
| setting_inputs = [ | |
| static_image_mode, | |
| max_num_hands, | |
| model_complexity, | |
| min_detection_conf, | |
| min_tracking_conf, | |
| selected_connections, | |
| draw_background, | |
| flip_image, | |
| ] | |
| outputs = [processed_image, multi_handedness, left_hands, right_hands] | |
| submit_uploaded_image.click(fn=process_image, inputs=[uploaded_image, *setting_inputs], outputs=outputs) | |
| submit_camera_picture.click(fn=process_image, inputs=[camera_picture, *setting_inputs], outputs=outputs) | |
| demo.launch() | |