WIP

.gitignore

@@ -3,3 +3,4 @@ lib
 output
 share
 input_img.jpg
+input_img.jpg

=1.12

@@ -1,14 +1,14 @@
 Requirement already satisfied: xtcocotools in ./lib/python3.10/site-packages (1.14.3)
-Requirement already satisfied: matplotlib>=2.1.0 in ./lib/python3.10/site-packages (from xtcocotools) (3.7.4)
+Requirement already satisfied: numpy>=1.20.0 in ./lib/python3.10/site-packages (from xtcocotools) (1.23.0)
 Requirement already satisfied: setuptools>=18.0 in ./lib/python3.10/site-packages (from xtcocotools) (65.5.0)
 Requirement already satisfied: cython>=0.27.3 in ./lib/python3.10/site-packages (from xtcocotools) (3.0.7)
-Requirement already satisfied: numpy>=1.20.0 in ./lib/python3.10/site-packages (from xtcocotools) (1.23.0)
+Requirement already satisfied: matplotlib>=2.1.0 in ./lib/python3.10/site-packages (from xtcocotools) (3.7.4)
+Requirement already satisfied: cycler>=0.10 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (0.12.1)
 Requirement already satisfied: fonttools>=4.22.0 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (4.47.0)
-Requirement already satisfied: python-dateutil>=2.7 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (2.8.2)
+Requirement already satisfied: contourpy>=1.0.1 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (1.2.0)
 Requirement already satisfied: packaging>=20.0 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (23.2)
-Requirement already satisfied: kiwisolver>=1.0.1 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (1.4.5)
-Requirement already satisfied: cycler>=0.10 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (0.12.1)
 Requirement already satisfied: pyparsing>=2.3.1 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (2.4.5)
-Requirement already satisfied: contourpy>=1.0.1 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (1.2.0)
+Requirement already satisfied: kiwisolver>=1.0.1 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (1.4.5)
+Requirement already satisfied: python-dateutil>=2.7 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (2.8.2)
 Requirement already satisfied: pillow>=6.2.0 in ./lib/python3.10/site-packages (from matplotlib>=2.1.0->xtcocotools) (9.4.0)
 Requirement already satisfied: six>=1.5 in ./lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=2.1.0->xtcocotools) (1.16.0)

app.py

@@ -1,4 +1,6 @@
-
+from keypoints_extraction import predict_pose
+from calculate_measures import calculate_all_measures
+from calculate_masks import calculate_seg_mask
 
 import os
 os.system("pip install xtcocotools>=1.12")
@@ -7,75 +9,27 @@ os.system("pip install 'mmcv>=2.0.0rc4,<2.1.0'")
 os.system("pip install 'mmdet>=3.0.0,<4.0.0'")
 os.system("pip install 'mmpose'")
 
-import PIL
-import cv2
-import numpy as np
-
-import torch
-from mmpose.apis import MMPoseInferencer
-from mmpose.apis import inference_topdown, init_model
-from mmpose.utils import register_all_modules
-register_all_modules()
-
 import gradio as gr
 
-import warnings
-
-warnings.filterwarnings("ignore")
-
-def save_image(img, img_path):
-    # Convert PIL image to OpenCV image
-    img = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
-    # Save OpenCV image
-    cv2.imwrite(img_path, img)
-
-def predict_pose(img):
-    img_path = "input_img.jpg"
-    save_image(img, img_path)
+def generate_output(front_img, side_img):
+    # TODO: These file names will need to be unique in case of multiple requests at once, and they will need to be deleted after the function is done.
+    front_keypoint_result = predict_pose(front_img, "front.jpg")
+    side_keypoint_result = predict_pose(side_img, "side.jpg")
 
-    result = mmpose_coco(img_path)
-    keypoints = result[0].pred_instances['keypoints'][0]
+    front_seg_mask = calculate_seg_mask(front_img)
+    side_rcnn_mask = calculate_seg_mask(side_img)
 
-    # Create a dictionary to store keypoints and their names
-    keypoints_data = {
-        'keypoints': keypoints.tolist(),
-        'keypoint_names': [
-            'nose',
-            'left_eye',
-            'right_eye',
-            'left_ear',
-            'right_ear',
-            'left_shoulder',
-            'right_shoulder',
-            'left_elbow',
-            'right_elbow',
-            'left_wrist',
-            'right_wrist',
-            'left_hip',
-            'right_hip',
-            'left_knee',
-            'right_knee',
-            'left_ankle',
-            'right_ankle'
-        ]
-    }
-    return (img, keypoints_data)
+    measures = calculate_all_measures(front_keypoint_result, side_keypoint_result, front_seg_mask, side_rcnn_mask)
 
-def mmpose_coco(img_path, 
-                config_file = 'mmpose/td-hm_hrnet-w48_8xb32-210e_coco-256x192.py', 
-                checkpoint_file = 'mmpose/td-hm_hrnet-w48_8xb32-210e_coco-256x192-0e67c616_20220913.pth'):
-    device = torch.cuda.current_device() if torch.cuda.is_available() else 'cpu'
-    # coco keypoints: 
-    # https://github.com/open-mmlab/mmpose/blob/master/mmpose/datasets/datasets/top_down/topdown_coco_dataset.py#L28
-    model = init_model(config_file, checkpoint_file, device=device)
-    results = inference_topdown(model, img_path)
-    return results
+    return (front_keypoint_result[0], front_keypoint_result[1], side_keypoint_result[0], side_keypoint_result[1])
 
-# download_test_image()
-input_image = gr.inputs.Image(type='pil', label="Original Image")
-output_image = gr.outputs.Image(type="pil", label="Output Image")
-output_text = gr.outputs.Textbox(label="Output Text")
+input_image_front = gr.inputs.Image(type='pil', label="Front Image")
+input_image_side = gr.inputs.Image(type='pil', label="Side Image")
+output_image_front = gr.outputs.Image(type="pil", label="Front Output Image")
+output_text_front = gr.outputs.Textbox(label="Front Output Text")
+output_image_side = gr.outputs.Image(type="pil", label="Front Output Image")
+output_text_side = gr.outputs.Textbox(label="Side Output Text")
 
 title = "MMPose detection for ShopByShape"
-iface = gr.Interface(fn=predict_pose, inputs=[input_image], outputs=[output_image, output_text], title=title)
+iface = gr.Interface(fn=generate_output, inputs=[input_image_front, input_image_side], outputs=[output_image_front, output_text_front, output_image_side, output_text_side], title=title)
 iface.launch()

calculate_masks.py

@@ -0,0 +1,32 @@
+from PIL import Image
+from transformers import SegformerImageProcessor, AutoModelForSemanticSegmentation
+import torch.nn as nn
+
+def calculate_seg_mask(image):
+    image = Image.open(image).convert("RGB")
+
+    processor = SegformerImageProcessor.from_pretrained("mattmdjaga/segformer_b2_clothes")
+    model = AutoModelForSemanticSegmentation.from_pretrained("mattmdjaga/segformer_b2_clothes")
+
+    class_names = {
+        0: "Background", 1: "Hat", 2: "Hair", 3: "Sunglasses", 
+        4: "Upper-clothes", 5: "Skirt", 6: "Pants", 7: "Dress", 
+        8: "Belt", 9: "Left-shoe", 10: "Right-shoe", 11: "Face", 
+        12: "Left-leg", 13: "Right-leg", 14: "Left-arm", 15: "Right-arm", 
+        16: "Bag", 17: "Scarf"
+    }
+
+    inputs = processor(images=image, return_tensors="pt")
+
+    outputs = model(**inputs)
+    logits = outputs.logits.cpu()
+
+    upsampled_logits = nn.functional.interpolate(
+        logits,
+        size=image.size[::-1],
+        mode="bilinear",
+        align_corners=False,
+    )
+
+    pred_seg = upsampled_logits.argmax(dim=1)[0]
+    return pred_seg

calculate_measures.py

@@ -0,0 +1,104 @@
+import numpy as np
+
+def get_width(mask, keypoint_y):
+    pred_np = mask.numpy()
+    # Find the index of the first occurrence of the mask label
+    first_index = np.argmax(pred_np[keypoint_y,:] > 0)
+
+    # Find the index of the last occurrence of the mask label
+    last_index = len(pred_np[keypoint_y,:]) - np.argmax(np.flip(pred_np[keypoint_y,:]) > 0) - 1
+
+    return last_index-first_index
+
+def calculate_all_measures(front_keypoints, side_keypoints, front_seg_mask, side_rcnn_mask):
+    results_dict = {}
+
+    # calculate the body length
+    # pick the longer from the two ankle keypoints on y coordinate
+    side_body_length = side_keypoints[15][1] if side_keypoints[15][1] > side_keypoints[16][1] else side_keypoints[16][1]
+
+    #     5: 'left_shoulder'
+    left_shoulder_y = round(front_keypoints[5][1])
+    # print("shoulder width", get_width(front_seg_mask, left_shoulder_y))
+    results_dict['shoulder_width'] = get_width(front_seg_mask, left_shoulder_y)
+
+    # remove left-arm mask
+    front_seg_mask[front_seg_mask == 14] = 0
+
+    # remove right-arm mask
+    front_seg_mask[front_seg_mask == 15] = 0
+
+    #     11: 'left_hip'
+    left_hip_y = round(front_keypoints[11][1])
+    # print("hip width", get_width(front_seg_mask, left_hip_y))
+    results_dict['hip_width'] = get_width(front_seg_mask, left_hip_y)
+
+    # calculate shoulder_to_hip distance
+    shoulder_to_hip_distance = front_keypoints[11][1] - front_keypoints[5][1]
+    # print("Shoulder to hip distance:", shoulder_to_hip_distance)
+    results_dict['shoulder_to_hip_distance'] = shoulder_to_hip_distance
+
+    # calculate hip_to_ankle distance
+    hip_to_ankle_distance = front_keypoints[16][1] - front_keypoints[12][1]
+    # print("Hip to ankle distance:", hip_to_ankle_distance)
+    results_dict['hip_to_ankle_distance'] = hip_to_ankle_distance
+
+    # calculate torso_to_leg proportions
+    torso_to_leg_proportions = shoulder_to_hip_distance / hip_to_ankle_distance
+    # print("Torso to leg proportions:", torso_to_leg_proportions)
+    results_dict['torso_to_leg_ratio'] = torso_to_leg_proportions
+
+    # waist 
+    # assuming waistline is x % higher from hips 
+    # hip_y axis - (40 % of shoulder to hip distance)
+    waist_y = round(front_keypoints[11][1] - (shoulder_to_hip_distance * 0.40))
+    # print("waist width", get_width(front_seg_mask, waist_y))
+    results_dict['waist_width'] = get_width(front_seg_mask, waist_y)
+
+    # Calculate bounding box for thigh
+    # right_knee = side_keypoints[side_keypoint_names.index("right_knee")]
+    # right_hip = side_keypoints[side_keypoint_names.index("right_hip")]
+    # # Calculate bounding box for torso
+    # right_shoulder = side_keypoints[side_keypoint_names.index("right_shoulder")]
+
+    # # Replace keypoints, keypoint_names, combined_mask, and original_image with your actual data
+    # thigh_bbox = calculate_bbox(side_original_image, right_knee, right_hip)
+    # torso_bbox = calculate_bbox(side_original_image, right_hip, right_shoulder)
+    # # Calculate midpoint coordinates
+    # torso_midpoint = [0, (right_hip[1] + right_shoulder[1]) / 2]
+    # lower_torso_bbox = calculate_bbox(side_original_image, right_hip, torso_midpoint)
+    # upper_torso_bbox = calculate_bbox(side_original_image, torso_midpoint, right_shoulder)
+
+    # # Replace keypoints, keypoint_names, combined_mask, and original_image with your actual data
+    # thigh_area = get_volume_result(side_rcnn_mask, side_original_image, thigh_bbox[1], thigh_bbox[0], thigh_bbox[3], thigh_bbox[2]) # Thigh volume
+    # torso_area = get_volume_result(side_rcnn_mask, side_original_image, torso_bbox[1], torso_bbox[0], torso_bbox[3], torso_bbox[2]) # Torso volume
+    # lower_torso_area = get_volume_result(side_rcnn_mask, side_original_image, lower_torso_bbox[1], lower_torso_bbox[0], lower_torso_bbox[3], lower_torso_bbox[2]) # Lower torso volume
+    # upper_torso_area = get_volume_result(side_rcnn_mask, side_original_image, upper_torso_bbox[1], upper_torso_bbox[0], upper_torso_bbox[3], upper_torso_bbox[2]) # Upper torso volume
+    # full_side_body_area = (side_rcnn_mask > 0).sum()
+    # # print(f"Thigh area: {thigh_area}")
+    # # print(f"Torso area: {torso_area}")
+    # # print(f"Lower torso area: {lower_torso_area}")
+    # # print(f"Upper torso area: {upper_torso_area}")
+    # results_dict['thigh_area'] = thigh_area
+    # results_dict['torso_area'] = torso_area
+    # results_dict['lower_torso_area'] = lower_torso_area
+    # results_dict['upper_torso_area'] = upper_torso_area
+    # results_dict['full_side_body_area'] = full_side_body_area
+
+    # # calculate ratios
+    # results_dict['thigh_normalised'] = thigh_area / side_body_length
+    # results_dict['torso_normalised'] = torso_area / side_body_length
+    # results_dict['thigh_to_torso_ratio_normalised'] = results_dict['thigh_normalised'] / results_dict['torso_normalised']
+    # results_dict['thigh_to_torso_ratio'] = thigh_area / torso_area
+    # results_dict['upper_torso_normalised'] = upper_torso_area / side_body_length
+    # results_dict['lower_torso_normalised'] = lower_torso_area / side_body_length
+    # results_dict['upper_to_lower_torso_normalised_ratio'] =  results_dict['upper_torso_normalised'] / results_dict['lower_torso_normalised']
+    # results_dict['upper_to_lower_torso_ratio'] = upper_torso_area / lower_torso_area
+    # results_dict['shoulder_to_hip_ratio'] = results_dict['shoulder_width'] / results_dict['hip_width']
+    # results_dict['shoulder_to_waist_ratio'] = results_dict['shoulder_width'] / results_dict['waist_width']
+    # results_dict['waist_to_hip_ratio'] = results_dict['waist_width'] / results_dict['hip_width']
+    # results_dict['thigh_to_body_ratio'] = thigh_area / full_side_body_area
+    # results_dict['upper_torso_to_body_ratio'] = upper_torso_area / full_side_body_area
+    # results_dict['upper_torso_to_body_ratio'] = upper_torso_area / full_side_body_area
+
+    return dict

keypoints_extraction.py

@@ -0,0 +1,56 @@
+import PIL
+import cv2
+import numpy as np
+
+import torch
+from mmpose.apis import MMPoseInferencer
+from mmpose.apis import inference_topdown, init_model
+from mmpose.utils import register_all_modules
+register_all_modules()
+
+def save_image(img, img_path):
+    # Convert PIL image to OpenCV image
+    img = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+    # Save OpenCV image
+    cv2.imwrite(img_path, img)
+
+def predict_pose(img, img_path):
+    save_image(img, img_path)
+
+    result = mmpose_coco(img_path)
+    keypoints = result[0].pred_instances['keypoints'][0]
+
+    # Create a dictionary to store keypoints and their names
+    keypoints_data = {
+        'keypoints': keypoints.tolist(),
+        'keypoint_names': [
+            'nose',
+            'left_eye',
+            'right_eye',
+            'left_ear',
+            'right_ear',
+            'left_shoulder',
+            'right_shoulder',
+            'left_elbow',
+            'right_elbow',
+            'left_wrist',
+            'right_wrist',
+            'left_hip',
+            'right_hip',
+            'left_knee',
+            'right_knee',
+            'left_ankle',
+            'right_ankle'
+        ]
+    }
+    return (img, keypoints_data)
+
+def mmpose_coco(img_path, 
+                config_file = 'mmpose/td-hm_hrnet-w48_8xb32-210e_coco-256x192.py', 
+                checkpoint_file = 'mmpose/td-hm_hrnet-w48_8xb32-210e_coco-256x192-0e67c616_20220913.pth'):
+    device = torch.cuda.current_device() if torch.cuda.is_available() else 'cpu'
+    # coco keypoints: 
+    # https://github.com/open-mmlab/mmpose/blob/master/mmpose/datasets/datasets/top_down/topdown_coco_dataset.py#L28
+    model = init_model(config_file, checkpoint_file, device=device)
+    results = inference_topdown(model, img_path)
+    return results