Upload 16 files

.gitattributes

@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+01_10.mp4 filter=lfs diff=lfs merge=lfs -text
+01_11.mp4 filter=lfs diff=lfs merge=lfs -text
+01_140.mp4 filter=lfs diff=lfs merge=lfs -text
+01_16.mp4 filter=lfs diff=lfs merge=lfs -text
+01_33.mp4 filter=lfs diff=lfs merge=lfs -text
+01_40.mov filter=lfs diff=lfs merge=lfs -text
+01_76.mp4 filter=lfs diff=lfs merge=lfs -text

01_10.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c6723fdcc6e1749ec60a76357db49dfcd4d152e6546c1927614854cc0b990f3
+size 1303693

01_11.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac78a8b402e1eb8be9a2d57e74f239677c4f70649476ad556e83c0851b94e77c
+size 2621440

01_140.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3bbec0ec1ede19c4d93208ece51bf4c08f7889864a8adb31a4d9535f4cc0599
+size 2621440

01_16.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f708564d62eb74888e5d62c851b5c63cdda6e6dc2c2d3a6f04f6605e3b8263c6
+size 2474745

01_33.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26a28bb4634266170a39c7828f3f7d0017df151a828e77a491cf58dd2eaf814f
+size 2621440

01_40.mov

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a7468bc1e790f101608b0648604573fa5e76e89dc861f9a05d03a8538174a8d
+size 2621440

01_76.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfde107e6b7b4b9e813a963b668ba772bed5d4bf16633cf0fb350b6ea5318d6c
+size 2621440

dive_recognition_functions.py

@@ -0,0 +1,351 @@
+import pickle
+import numpy as np
+import os
+import math
+from scipy.signal import find_peaks
+from matplotlib import pyplot as plt
+
+def get_scale_factor(dive_data):
+    # distance between thorax and pelvis
+    distances = []
+    for pose_pred in dive_data['pose_pred']:
+        if pose_pred is not None:
+            distances.append(math.dist(pose_pred[0][6], pose_pred[0][7]))
+    distances.sort()
+    return np.median(distances)
+
+def find_angle(vector1, vector2):
+    unit_vector_1 = vector1 / np.linalg.norm(vector1)
+    unit_vector_2 = vector2 / np.linalg.norm(vector2)
+    dot_product = np.dot(unit_vector_1, unit_vector_2)
+    angle = math.degrees(np.arccos(dot_product))
+    return angle
+
+def is_back_facing(dive_data, board_side):
+    directions = []
+    for i in range(len(dive_data['pose_pred'])):
+        pose_pred = dive_data['pose_pred'][i]
+        if pose_pred is None or dive_data['above_boards'][i] == 0:
+            continue
+        pose_pred = pose_pred[0]
+        
+        ## left knee bend ###
+        l_knee = pose_pred[4]
+        l_ankle = pose_pred[5]
+        l_hip = pose_pred[3]
+        l_knee_ankle = [l_ankle[0] - l_knee[0], 0-(l_ankle[1] - l_knee[1])]
+        l_knee_hip = [l_hip[0] - l_knee[0], 0-(l_hip[1] - l_knee[1])]
+        l_direction = rotation_direction(l_knee_hip, l_knee_ankle)
+        
+        ## right knee bend ###
+        r_knee = pose_pred[1]
+        r_ankle = pose_pred[0]
+        r_hip = pose_pred[2]
+        r_knee_ankle = [r_ankle[0] - r_knee[0], 0-(r_ankle[1] - r_knee[1])]
+        r_knee_hip = [r_hip[0] - r_knee[0], 0-(r_hip[1] - r_knee[1])]
+        r_direction = rotation_direction(r_knee_hip, r_knee_ankle)
+        if l_direction == r_direction and l_direction != 0 and board_side == 'left':
+            # we're looking for more clockwise
+            return l_direction < 0
+        elif l_direction == r_direction and l_direction != 0:
+            # we're looking for more counterclockwise
+            return l_direction > 0
+    return False
+
+def rotation_direction(vector1, vector2, threshold=0.4):
+    # Calculate the determinant to determine rotation direction
+    determinant = vector1[0] * vector2[1] - vector1[1] * vector2[0]
+    mag1= np.linalg.norm(vector1)
+    mag2= np.linalg.norm(vector2)
+    norm_det = determinant/(mag1*mag2)
+    # print("determinant", determinant/(mag1*mag2))
+    # print(norm_det)
+    if norm_det > threshold:
+        # return "counterclockwise"
+        return 1
+    elif norm_det < 0-threshold:
+        # return "clockwise"
+        return -1
+    else:
+        # return "not determinent"
+        return 0
+
+def find_position(dive_data):
+    angles = []
+    three_in_a_row = 0
+    for i in range(1, len(dive_data['pose_pred'])):
+        pose_pred = dive_data['pose_pred'][i]
+        if pose_pred is None or dive_data['som'][i]==0:
+            continue
+        pose_pred = pose_pred[0]
+        l_knee = pose_pred[4]
+        l_ankle = pose_pred[5]
+        l_hip = pose_pred[3]
+        l_knee_ankle = [l_ankle[0] - l_knee[0], 0-(l_ankle[1] - l_knee[1])]
+        l_knee_hip = [l_hip[0] - l_knee[0], 0-(l_hip[1] - l_knee[1])]
+        angle = find_angle(l_knee_ankle, l_knee_hip)
+        angles.append(angle)
+        # print(angle)
+        if angle < 70:
+            three_in_a_row += 1
+            if three_in_a_row >=3:
+                return 'tuck'
+        else:
+            three_in_a_row =0
+    if twist_counter_full_dive(dive_data) > 0 and som_counter_full_dive(dive_data)[0] < 5:
+        return 'free'
+    return 'pike'
+
+
+def distance_point_to_line_segment(px, py, x1, y1, x2, y2):
+    # Calculate the squared distance from point (px, py) to the line segment [(x1, y1), (x2, y2)]
+    def sqr_distance_point_to_segment():
+        line_length_sq = (x2 - x1)**2 + (y2 - y1)**2
+        if line_length_sq == 0:
+            return (px - x1)**2 + (py - y1)**2
+        t = max(0, min(1, ((px - x1) * (x2 - x1) + (py - y1) * (y2 - y1)) / line_length_sq))
+        return ((px - (x1 + t * (x2 - x1)))**2 + (py - (y1 + t * (y2 - y1)))**2)
+
+    # Calculate the closest point on the line segment to the given point (px, py)
+    def closest_point_on_line_segment():
+        line_length_sq = (x2 - x1)**2 + (y2 - y1)**2
+        if line_length_sq == 0:
+            return x1, y1
+        t = max(0, min(1, ((px - x1) * (x2 - x1) + (py - y1) * (y2 - y1)) / line_length_sq))
+        closest_x = x1 + t * (x2 - x1)
+        closest_y = y1 + t * (y2 - y1)
+        return closest_x, closest_y
+
+    closest_point = closest_point_on_line_segment()
+    distance = math.sqrt(sqr_distance_point_to_segment())
+
+    return closest_point, distance
+
+def min_distance_from_line_to_circle(line_start, line_end, circle_center, circle_radius):
+    closest_point, distance = distance_point_to_line_segment(circle_center[0], circle_center[1],
+                                                             line_start[0], line_start[1],
+                                                             line_end[0], line_end[1])
+
+    min_distance = max(0, distance - circle_radius)
+    return min_distance
+
+
+def twister(pose_pred, prev_pose_pred=None, in_petal=False, petal_count=0, outer=10, inner=9, valid=17, middle=0.5):
+    if pose_pred is None:
+        return petal_count, in_petal
+    min_dist = 0
+    # Users/lokamoto/Comprehensive_AQA/output/joint_plots/FINAWorldChampionships2019_Women10m_final_r1_0
+    pose_pred = pose_pred[0]
+    vector1 = [pose_pred[2][0] - pose_pred[3][0], 0-(pose_pred[2][1] - pose_pred[3][1])]
+    if prev_pose_pred is not None:
+        prev_pose_pred = prev_pose_pred[0]
+        prev_pose_pred = [prev_pose_pred[2][0] - prev_pose_pred[3][0], 0-(prev_pose_pred[2][1] - prev_pose_pred[3][1])]
+        min_dist = min_distance_from_line_to_circle(prev_pose_pred, vector1, (0, 0), middle)
+    if np.linalg.norm(vector1) > valid:
+        return petal_count, in_petal
+    if min_dist is not None and in_petal and np.linalg.norm(vector1) > outer and min_dist == 0: #and np.linalg.norm(vector1) > 8
+        petal_count += 1
+        # print('leaving petal')
+        # print('going in new petal')
+    elif not in_petal and np.linalg.norm(vector1) > outer: #and min_dist > 3: #and np.linalg.norm(vector1) > 8
+        in_petal = True
+        petal_count += 1
+        # print('going in petal')
+    elif in_petal and np.linalg.norm(vector1) < inner:
+        in_petal = False
+        
+        # print('leaving petal')
+    # print(vector)
+    return petal_count, in_petal
+
+
+def twist_counter_full_dive(dive_data, visualize=False):
+    twists_gt = []
+    twists_gt.extend(range(29, 36))
+    twists_gt.extend([41, 42]) 
+    start = [1, 2, 3, 4, 5, 6, 7]
+    # key = ('01',17)
+    # print(ground_truth[key][0])
+    # print(dive_data['twist_counts'])
+    dist_hip = []
+    prev_pose_pred = None
+    in_petal=False
+    petal_count=0
+    scale = get_scale_factor(dive_data)
+    valid = scale / 1.5
+    outer = scale / 3.2
+    inner = scale / 3.4
+    middle = 0.5
+    next_next_pose_pred = dive_data['pose_pred'][4]
+    # bad_poses = invalid_poses(key)
+    for i in range(len(dive_data['pose_pred'])):
+        # if ground_truth[key][4][i] not in twists_gt:
+        #     continue
+        # if i in bad_poses:
+        #     continue
+        pose_pred = dive_data['pose_pred'][i]
+        if i < len(dive_data['pose_pred']) - 1:
+            next_pose_pred = dive_data['pose_pred'][i + 1]
+        if i < len(dive_data['pose_pred']) - 4 and next_next_pose_pred is not None:
+            next_next_pose_pred = dive_data['pose_pred'][i + 4]
+        if pose_pred is None or dive_data['on_boards'][i] == 1 or dive_data['position_tightness'][i] <= 80 or next_next_pose_pred is None:
+            continue
+        petal_count, in_petal = twister(pose_pred, prev_pose_pred=prev_pose_pred, in_petal=in_petal, petal_count=petal_count, outer=outer, inner=inner, middle=middle, valid=valid)
+        prev_pose_pred = pose_pred
+        # print(petal_count)
+        if visualize:
+            pose_pred = pose_pred[0]
+            dist_hip.append([pose_pred[2][0] - pose_pred[3][0], 0-(pose_pred[2][1] - pose_pred[3][1])])
+    if visualize:
+        dist_hip = np.array(dist_hip)
+        plt.plot(dist_hip[:, 0], dist_hip[:, 1], label="right-to-left hip")
+        circle1 = plt.Circle((0, 0), outer, fill=False)
+        plt.gca().add_patch(circle1)
+        circle2 = plt.Circle((0, 0), inner, fill=False)
+        plt.gca().add_patch(circle2)
+        circle3 = plt.Circle((0, 0), valid, fill=False)
+        plt.gca().add_patch(circle3)
+        plt.legend()
+        plt.show()
+    return petal_count
+
+def rotation_direction_som(vector1, vector2, threshold=0.4):
+    # Calculate the determinant to determine rotation direction
+    determinant = vector1[0] * vector2[1] - vector1[1] * vector2[0]
+    mag1= np.linalg.norm(vector1)
+    mag2= np.linalg.norm(vector2)
+    norm_det = determinant/(mag1*mag2)
+    theta = np.arcsin(norm_det)
+    return math.degrees(theta)
+
+def is_handstand(dive_data):
+    first_frame_pose_pred = dive_data['pose_pred'][0]
+    handstand = False
+    if first_frame_pose_pred[0][6][1] < first_frame_pose_pred[0][7][1]:
+        handstand = True
+    return handstand
+    
+def som_counter_full_dive(dive_data, visualize=False):
+    start = [1, 2, 3, 4, 5, 6, 7]
+    entry = [36]
+    half_som_count = 0
+    # key = ('04', 47)
+    # print(ground_truth[key][0])
+    dist_body = []
+    handstand = is_handstand(dive_data)
+    # key = ('04',42)
+    next_next_pose_pred = dive_data['pose_pred'][2]
+    # bad_poses = invalid_poses(key)
+    prev = None
+    for i in range(len(dive_data['pose_pred'])):
+        # if i in bad_poses:
+        #     continue
+        pose_pred = dive_data['pose_pred'][i]
+        if i < len(dive_data['pose_pred']) - 2 and next_next_pose_pred is not None:
+            next_next_pose_pred = dive_data['pose_pred'][i + 2]
+        if pose_pred is None or next_next_pose_pred is None or dive_data['on_boards'][i] == 1: # ground_truth[key][4][i] in start:
+            continue
+        
+        pose_pred = pose_pred[0]
+        vector1 = [pose_pred[7][0] - pose_pred[6][0], 0-(pose_pred[7][1] - pose_pred[6][1])] # flip y axis
+        if (not handstand and half_som_count % 2 == 0) or (handstand and half_som_count % 2 == 1):  
+            vector2 = [0, -1]
+        else:
+            vector2 = [0, 1]        
+        sensitivity = 115
+        if prev is not None and find_angle(vector1, prev) > sensitivity:
+            continue
+        is_clockwise = is_rotating_clockwise(dive_data)
+        if prev is not None and ((is_clockwise and rotation_direction_som(vector1, prev)<0) or (not is_clockwise and rotation_direction_som(vector1, prev)>0)):
+            continue
+        angle = find_angle(vector1, vector2)
+        # print("unit_vector_1:", unit_vector_1)
+        # print("looking for vector:", vector2)
+        # print(half_som_count)
+        # print(angle)
+        if angle <= 75:
+            half_som_count += 1
+        if visualize:
+            dist_body.append([pose_pred[7][0] - pose_pred[6][0], 0-(pose_pred[7][1] - pose_pred[6][1])])
+        prev = vector1
+            
+    if visualize:
+        dist_body = np.array(dist_body)
+        plt.plot(dist_body[:, 0], dist_body[:, 1], label="pelvis-to-thorax")
+        plt.xlabel("x-coord")
+        plt.ylabel("y-coord")
+
+        plt.legend()
+        plt.show()
+    return half_som_count, handstand
+
+def getDiveInfo(diveNum):
+    handstand = (diveNum[0] == '6')
+    expected_som = int(diveNum[2])
+    if len(diveNum) == 5:
+        expected_twists = int(diveNum[3])
+    else:
+        expected_twists = 0
+    if diveNum[0] == '1' or diveNum[0] == '3' or diveNum[:2] == '51' or diveNum[:2] == '53' or diveNum[:2] == '61' or diveNum[:2] == '63':
+        back_facing = False
+    else:
+        back_facing = True
+    if diveNum[0] == '1' or diveNum[:2] == '51' or diveNum[:2] == '61':
+        expected_direction = 'front'
+    elif diveNum[0] == '2' or diveNum[:2] == '52' or diveNum[:2] == '62':
+        expected_direction = 'back'
+    elif diveNum[0] == '3' or diveNum[:2] == '53' or diveNum[:2] == '63':
+        expected_direction = 'reverse'
+    elif diveNum[0] == '4':
+        expected_direction = 'inward'
+    if diveNum[-1] == 'b':
+        position = 'pike'
+    elif diveNum[-1] == 'c':
+        position = 'tuck'
+    else:
+        position = 'free'
+    return handstand, expected_som, expected_twists, back_facing, expected_direction, position
+
+def get_direction(dive_data):
+    clockwise = is_rotating_clockwise(dive_data)
+    board_side = dive_data['board_side']
+    if board_side == "right":
+        back_facing = is_back_facing(dive_data, 'right')
+        if back_facing and clockwise:
+            direction = 'inward'
+        elif back_facing and not clockwise:
+            direction = 'back'
+        elif not back_facing and clockwise:
+            direction = 'reverse'
+        elif not back_facing and not clockwise:
+            direction = 'front'
+    else:
+        back_facing = is_back_facing(dive_data, 'left')
+        if back_facing and clockwise:
+            direction = 'back'
+        elif back_facing and not clockwise:
+            direction = 'inward'
+        elif not back_facing and clockwise:
+            direction = 'front'
+        elif not back_facing and not clockwise:
+            direction = 'reverse'
+    return direction
+
+def is_rotating_clockwise(dive_data):
+    directions = []
+    for i in range(1, len(dive_data['pose_pred'])):
+        if dive_data['pose_pred'][i] is None or dive_data['pose_pred'][i-1] is None:
+            continue
+        if dive_data['on_boards'][i] == 0:
+            prev_pose_pred_hip = dive_data['pose_pred'][i-1][0][3]
+            curr_pose_pred_hip = dive_data['pose_pred'][i][0][3]
+            prev_pose_pred_knee = dive_data['pose_pred'][i-1][0][4]
+            curr_pose_pred_knee = dive_data['pose_pred'][i][0][4]   
+            prev_hip_knee = [prev_pose_pred_knee[0] - prev_pose_pred_hip[0], 0-(prev_pose_pred_knee[1] - prev_pose_pred_hip[1])]
+            curr_hip_knee = [curr_pose_pred_knee[0] - curr_pose_pred_hip[0], 0-(curr_pose_pred_knee[1] - curr_pose_pred_hip[1])]
+            direction = rotation_direction(prev_hip_knee, curr_hip_knee, threshold=0)
+            directions.append(direction)
+    return np.sum(directions) < 0
+
+
+

generate_reports.py

@@ -0,0 +1,551 @@
+from jinja2 import Environment, FileSystemLoader
+import pickle
+import os
+import numpy as np
+from PIL import Image, ImageDraw
+from io import BytesIO
+import cv2
+import base64
+from pathlib import Path
+import torch
+import gradio as gr
+
+############ Generate GIF functions ###################################################################################
+def generate_gif(local_directory, image_names, speed_factor=1, loop=0):
+    """
+    Generate a GIF from a sequence of images paths saved in a local directory.
+
+    Parameters:
+    - local_directory (str): Directory path where the images are located
+    - image_paths (list): List of filenames of the input images.
+    - speed_factor (int): How fast the GIF is, the higher the less the delay is between frames
+    - loop (int): Number of loops (0 for infinite loop).
+
+    Returns:
+    - Bytes of GIF
+    """
+    images = []
+    durations = []
+    for image_name in image_names:
+        img = Image.open(os.path.join(local_directory, image_name))
+        images.append(img)
+        try:
+            duration = img.info['duration']
+        except KeyError:
+            duration = 100  # Default duration in case 'duration' is not available
+        durations.append(duration)
+
+    # Calculate the adjusted durations based on the speed factor
+    adjusted_durations = [int(duration / speed_factor) for duration in durations]
+
+    # Save as GIF to an in-memory buffer
+    gif_buffer = BytesIO()
+    images[0].save(gif_buffer, format='GIF', save_all=True, append_images=images[1:], duration=adjusted_durations, loop=loop)
+
+    # Get the content of the buffer as bytes
+    gif_content = gif_buffer.getvalue()
+    gif_content = base64.b64encode(gif_content).decode('utf-8')
+    return gif_content
+
+def generate_gif_from_frames(frames, speed_factor=1, loop=0, progress=gr.Progress()):
+    """
+    Generate a GIF from a sequence of images.
+
+    Parameters:
+    - frames (list): List of cv2 frames
+    - speed_factor (int): How fast the GIF is, the higher the less the delay is between frames
+    - loop (int): Number of loops (0 for infinite loop).
+
+    Returns:
+    - Bytes of GIF
+    """
+    durations = []
+    images = []
+    i = 0
+    for frame in frames:
+        # progress((len(frames)+i)/(2*len(frames)), desc="Abstracting Symbols")
+        image = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) 
+        images.append(image)
+        duration = 100  # Default duration in case 'duration' is not available
+        durations.append(duration)
+        i+=1
+
+    # Calculate the adjusted durations based on the speed factor
+    adjusted_durations = [int(duration / speed_factor) for duration in durations]
+
+    # images[0].save('test.gif', 
+    #         save_all = True, append_images = images[1:], 
+    #         optimize = False, duration =adjusted_durations, loop=loop) 
+
+    # Save as GIF to an in-memory buffer
+    gif_buffer = BytesIO()
+    images[0].save(gif_buffer, format='GIF', save_all=True, append_images=images[1:], duration=adjusted_durations, loop=loop)
+
+    # Get the content of the buffer as bytes
+    gif_content = gif_buffer.getvalue()
+    gif_content = base64.b64encode(gif_content).decode('utf-8')
+    return gif_content
+
+##########################################################################################################
+############ Overlay Symbols on Frames  ######################################################################
+SKELETON = [
+    [1,2],[1,0],[2,6],[3,6],[4,5],[3,4],[6,7],[7,8],[9,8],[7,12],[7,13],[11,12],[13,14],[14,15],[10,11]
+]
+CocoColors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0],
+              [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255],
+              [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]]
+NUM_KPTS = 16
+def draw_pose(keypoints,img, board_end_coord):
+    """draw the keypoints and the skeletons.
+    :params keypoints: the shape should be equal to [17,2]
+    :params img:
+    """
+    assert keypoints.shape == (NUM_KPTS,2)
+    for i in range(len(SKELETON)):
+        kpt_a, kpt_b = SKELETON[i][0], SKELETON[i][1]
+        x_a, y_a = keypoints[kpt_a][0],keypoints[kpt_a][1]
+        x_b, y_b = keypoints[kpt_b][0],keypoints[kpt_b][1] 
+        cv2.circle(img, (int(x_a), int(y_a)), 6, CocoColors[i], -1)
+        cv2.circle(img, (int(x_b), int(y_b)), 6, CocoColors[i], -1)
+        cv2.line(img, (int(x_a), int(y_a)), (int(x_b), int(y_b)), CocoColors[i], 2)
+    # cv2.circle(img, board_end_coord, 4, [0, 0, 0], -1)
+
+def draw_symbols(opencv_image, pose_preds, board_end_coord, plat_outputs, splash_pred_mask, above_board=None):
+    # Open image using openCV2 
+    # opencv_image = cv2.imread(image_path) 
+    if pose_preds is not None:
+        draw_pose(np.array(pose_preds)[0],opencv_image, board_end_coord)
+    if above_board is None or above_board==1:
+        draw_platform(opencv_image, plat_outputs)
+    draw_splash(opencv_image, splash_pred_mask)
+    # # Notice the COLOR_BGR2RGB which means that the color is 
+    # # converted from BGR to RGB 
+    # color_converted = cv2.cvtColor(opencv_image, cv2.COLOR_BGR2RGB) 
+
+    # # Displaying the converted image 
+    # pil_image = Image.fromarray(color_converted)
+    return opencv_image
+
+def draw_platform(opencv_image, output):
+    pred_classes = output['instances'].pred_classes.cpu().numpy()
+    platforms = np.where(pred_classes == 0)[0]
+    scores = output['instances'].scores[platforms]
+    if len(scores) == 0:
+      return
+    pred_masks = output['instances'].pred_masks[platforms]
+    max_instance = torch.argmax(scores)
+    pred_mask = np.array(pred_masks[max_instance].cpu()) 
+    # Convert the mask to a binary image    
+    binary_mask = pred_mask.squeeze().astype(np.uint8)
+    contours, hierarchy = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
+
+    cv2.drawContours(opencv_image, contours[0], -1, (36, 255, 12), thickness=5)
+
+def draw_splash(opencv_image, pred_mask):
+    # Convert the mask to a binary image  
+    if pred_mask is None:
+        return  
+    binary_mask = pred_mask.squeeze().astype(np.uint8)
+    contours, hierarchy = cv2.findContours(binary_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
+
+    cv2.drawContours(opencv_image, contours[0], -1, (0, 0, 0), thickness=5)
+
+##########################################################################################################
+############ Generate HTML template ######################################################################
+
+def generate_report(template_path, data, directory, local_directory, progress=gr.Progress()):
+    # Load the template environment
+    env = Environment(loader=FileSystemLoader('.'))
+
+    file_names = os.listdir(local_directory)
+    file_names.sort()
+    file_names = np.array(file_names)
+    progress(0.9, desc="Generating Score Report")
+    # Load the template
+    is_twister = data['twist_position_tightness']['raw_score'] is not None
+    overall_score_desc = data['overall_score']['description']
+    overall_score = '%.1f' % data['overall_score']['raw_score']
+
+    feet_apart_score = round(float('%.2f' % data['feet_apart']['raw_score']))
+    feet_apart_peaks = file_names[data['feet_apart']['peaks']]
+    feet_apart_gif = None
+    has_feet_apart_peaks = False
+    if len(feet_apart_peaks) > 0:
+        has_feet_apart_peaks = True
+        feet_apart_gif = generate_gif(local_directory, feet_apart_peaks, speed_factor = 0.05)
+    feet_apart_percentile = round(float('%.2f' % (data['feet_apart']['percentile'] *100)))
+    feet_apart_percentile_divided_by_ten = '%.1f' % (data['feet_apart']['percentile'] *10)
+
+    include_height_off_platform = False
+    height_off_board_score = data['height_off_board']['raw_score']
+    height_off_board_percentile = data['height_off_board']['percentile']
+    height_off_board_description = None
+    height_off_board_percentile_divided_by_ten = None
+    encoded_height_off_board_frame = None
+    if height_off_board_score is not None:
+        include_height_off_platform = True
+        height_off_board_score = round(float('%.2f' % data['height_off_board']['raw_score']))
+        height_off_board_frame = file_names[data['height_off_board']['frame_index']]
+        height_off_board_frame_path = os.path.join(local_directory, height_off_board_frame)
+        with open(height_off_board_frame_path, "rb") as image_file:
+            encoded_height_off_board_frame = base64.b64encode(image_file.read()).decode('utf-8')
+        height_off_board_percentile = round(float('%.2f' % (data['height_off_board']['percentile'] *100)))
+        height_off_board_percentile_divided_by_ten = '%.1f' % (data['height_off_board']['percentile'] *10)
+        if float(height_off_board_percentile_divided_by_ten) > 5:
+            height_off_board_description = "good"
+        else:
+            height_off_board_description = "a bit on the lower side"
+
+    dist_from_board_score = '%.2f' % data['distance_from_board']['raw_score']
+    dist_from_board_frame = file_names[data['distance_from_board']['frame_index']]
+    dist_from_board_frame_path = os.path.join(local_directory, dist_from_board_frame)
+    with open(dist_from_board_frame_path, "rb") as image_file:
+        encoded_dist_from_board_frame = base64.b64encode(image_file.read()).decode('utf-8')
+    dist_from_board_percentile = data['distance_from_board']['percentile']
+    if 'good' in dist_from_board_percentile:
+        dist_from_board_percentile_status = "Good"
+    elif 'far' in dist_from_board_percentile:
+        dist_from_board_percentile_status = "Too Far"
+    else:
+        dist_from_board_percentile_status = "Too Close"
+
+    knee_bend_score = data['knee_bend']['raw_score']
+    knee_bend_percentile = data['knee_bend']['percentile']
+    knee_bend_frames = []
+    knee_bend_percentile_divided_by_ten = None
+    if knee_bend_score is not None:
+        knee_bend_score = round(float('%.2f' % (data['knee_bend']['raw_score'])))
+        knee_bend_frames = file_names[data['knee_bend']['frame_indices']]
+        knee_bend_percentile = round(float('%.2f' % (knee_bend_percentile * 100)))
+        knee_bend_percentile_divided_by_ten = '%.1f' % (data['knee_bend']['percentile'] * 10)
+    
+    som_position_tightness_score = data['som_position_tightness']['raw_score']
+    som_position_tightness_percentile = data['som_position_tightness']['percentile']
+    som_position_tightness_position = data['som_position_tightness']['position']
+    som_position_tightness_frames = file_names[data['som_position_tightness']['frame_indices']]
+    som_position_tightness_gif = None
+    som_position_tightness_percentile_divided_by_ten = None
+    if som_position_tightness_score is not None:
+        if is_twister:
+            som_position_tightness_score = round(float('%.2f' % (data['som_position_tightness']['raw_score'] + 15)))
+        else:
+            som_position_tightness_score = round(float('%.2f' % (data['som_position_tightness']['raw_score'])))
+        som_position_tightness_gif = generate_gif(local_directory, som_position_tightness_frames)
+        som_position_tightness_percentile = round(float('%.2f' % (som_position_tightness_percentile * 100)))
+        som_position_tightness_percentile_divided_by_ten = '%.1f' % (data['som_position_tightness']['percentile'] * 10)
+    twist_position_tightness_score = data['twist_position_tightness']['raw_score']
+    twist_position_tightness_frames = []
+    twist_position_tightness_gif = None
+    twist_position_tightness_percentile = None
+    twist_position_tightness_percentile_divided_by_ten = None
+    if twist_position_tightness_score is not None:
+        twist_position_tightness_score = round(float('%.2f' % twist_position_tightness_score))
+        twist_position_tightness_frames = file_names[data['twist_position_tightness']['frame_indices']]
+        twist_position_tightness_gif = generate_gif(local_directory, twist_position_tightness_frames)
+        twist_position_tightness_percentile = round(float('%.2f' % (data['twist_position_tightness']['percentile'] * 100)))
+        twist_position_tightness_percentile_divided_by_ten = '%.1f' % (data['twist_position_tightness']['percentile'] * 10)
+    over_under_rotation_score = round(float('%.2f' % data['over_under_rotation']['raw_score']))
+    over_under_rotation_frame = file_names[data['over_under_rotation']['frame_index']]
+    over_under_rotation_percentile = round(float('%.2f' % (data['over_under_rotation']['percentile'] * 100)))
+    over_under_rotation_percentile_divided_by_ten = '%.1f' % (data['over_under_rotation']['percentile'] * 10)
+    straightness_during_entry_score = round(float('%.2f' % data['straightness_during_entry']['raw_score']))
+    straightness_during_entry_frames = file_names[data['straightness_during_entry']['frame_indices']]
+    straightness_during_entry_gif = generate_gif(local_directory, straightness_during_entry_frames, speed_factor = 0.5)
+    straightness_during_entry_percentile = round(float('%.2f' % (data['straightness_during_entry']['percentile'] * 100)))
+    straightness_during_entry_percentile_divided_by_ten = '%.1f' % (data['straightness_during_entry']['percentile'] * 10)
+    splash_score = round(float('%.2f' % data['splash']['raw_score']))
+    splash_frame = file_names[data['splash']['maximum_index']]
+    splash_indices = file_names[data['splash']['frame_indices']]
+    splash_gif = None
+    if len(splash_indices) > 0:
+        splash_gif = generate_gif(local_directory, splash_indices)
+    splash_percentile = round(float('%.2f' % (data['splash']['percentile'] * 100)))
+    splash_percentile_divided_by_ten = '%.1f' % (data['splash']['percentile'] * 10)
+    if float(splash_percentile) < 50:
+        splash_description = 'on the larger side'
+    else:
+        splash_description = 'small'
+    template = env.get_template(template_path)
+    data = {
+        'directory' : directory,
+        'local_directory': local_directory,
+        'is_twister' : is_twister,
+        'overall_score_desc' : overall_score_desc,
+        'overall_score' : overall_score,
+        'feet_apart_score' : feet_apart_score,
+        'feet_apart_peaks' : feet_apart_peaks,
+        'has_feet_apart_peaks' : has_feet_apart_peaks,
+        'feet_apart_gif' : feet_apart_gif,
+        'feet_apart_percentile' : feet_apart_percentile,
+        'feet_apart_percentile_divided_by_ten': feet_apart_percentile_divided_by_ten,
+
+        'include_height_off_platform': include_height_off_platform,
+        'height_off_board_score' : height_off_board_score,
+        'height_off_board_percentile' : height_off_board_percentile,
+        'encoded_height_off_board_frame' : encoded_height_off_board_frame,
+        'height_off_board_percentile_divided_by_ten': height_off_board_percentile_divided_by_ten,
+        'height_off_board_description' : height_off_board_description,
+
+        'dist_from_board_score' : dist_from_board_score,
+        'dist_from_board_frame' : dist_from_board_frame,
+        'encoded_dist_from_board_frame': encoded_dist_from_board_frame,
+        'dist_from_board_percentile' : dist_from_board_percentile,
+        'dist_from_board_percentile_status': dist_from_board_percentile_status,
+
+        'knee_bend_score' : knee_bend_score,
+        'knee_bend_frames' : knee_bend_frames,
+        'knee_bend_percentile' : knee_bend_percentile,
+        'knee_bend_percentile_divided_by_ten' : knee_bend_percentile_divided_by_ten,
+
+        'som_position_tightness_score' : som_position_tightness_score,
+        'som_position_tightness_frames' : som_position_tightness_frames,
+        'som_position_tightness_gif' : som_position_tightness_gif,
+        'som_position_tightness_position' : som_position_tightness_position,
+        'som_position_tightness_percentile' : som_position_tightness_percentile,
+        'som_position_tightness_percentile_divided_by_ten' : som_position_tightness_percentile_divided_by_ten,
+        'twist_position_tightness_score' : twist_position_tightness_score,
+        'twist_position_tightness_frames': twist_position_tightness_frames,
+        'twist_position_tightness_percentile' : twist_position_tightness_percentile,
+        'twist_position_tightness_percentile_divided_by_ten' : twist_position_tightness_percentile_divided_by_ten,
+        'twist_position_tightness_gif' : twist_position_tightness_gif,
+        'over_under_rotation_score' : over_under_rotation_score,
+        'over_under_rotation_frame' : over_under_rotation_frame,
+        'over_under_rotation_percentile' : over_under_rotation_percentile,
+        'over_under_rotation_percentile_divided_by_ten' : over_under_rotation_percentile_divided_by_ten,
+        'straightness_during_entry_score' : straightness_during_entry_score,
+        'straightness_during_entry_gif' : straightness_during_entry_gif,
+        'straightness_during_entry_percentile' : straightness_during_entry_percentile,
+        'straightness_during_entry_percentile_divided_by_ten': straightness_during_entry_percentile_divided_by_ten,
+        'splash_score' : splash_score,
+        'splash_frame' : splash_frame,
+        'splash_gif' : splash_gif,
+        'splash_percentile' : splash_percentile,
+        'splash_percentile_divided_by_ten': splash_percentile_divided_by_ten,
+        'splash_description' : splash_description,
+    }
+    # Render the template with the provided data
+    report_content = template.render(data)
+    return report_content
+
+
+def generate_report_from_frames(template_path, data, frames):
+    # Load the template environment
+    env = Environment(loader=FileSystemLoader('.'))
+
+    frames = np.array(frames)
+    # Load the template
+    is_twister = data['twist_position_tightness']['raw_score'] is not None
+    overall_score_desc = data['overall_score']['description']
+    overall_score = '%.1f' % data['overall_score']['raw_score']
+
+    feet_apart_score = round(float('%.2f' % data['feet_apart']['raw_score']))
+    feet_apart_peaks = frames[data['feet_apart']['peaks']]
+    feet_apart_gif = None
+    has_feet_apart_peaks = False
+    if len(feet_apart_peaks) > 0:
+        has_feet_apart_peaks = True
+        feet_apart_gif = generate_gif_from_frames(feet_apart_peaks, speed_factor = 0.05)
+    feet_apart_percentile = round(float('%.2f' % (data['feet_apart']['percentile'] *100)))
+    feet_apart_percentile_divided_by_ten = '%.1f' % (data['feet_apart']['percentile'] *10)
+
+    include_height_off_platform = False
+    height_off_board_score = data['height_off_board']['raw_score']
+    height_off_board_frame = None
+    height_off_board_percentile = None
+    height_off_board_percentile_divided_by_ten = None
+    height_off_board_description = None
+    encoded_height_off_board_frame = None
+    if height_off_board_score is not None:
+        include_height_off_platform = True
+        height_off_board_score = round(float('%.2f' % data['height_off_board']['raw_score']))
+        height_off_board_frame = Image.fromarray(cv2.cvtColor(frames[data['height_off_board']['frame_index']], cv2.COLOR_BGR2RGB))
+        height_buffer = BytesIO()
+        height_off_board_frame.save(height_buffer, format='JPEG')
+        encoded_height_off_board_frame = base64.b64encode(height_buffer.getvalue()).decode('utf-8')
+        height_off_board_percentile = round(float('%.2f' % (data['height_off_board']['percentile'] *100)))
+        height_off_board_percentile_divided_by_ten = '%.1f' % (data['height_off_board']['percentile'] *10)
+        if float(height_off_board_percentile_divided_by_ten) > 5:
+            height_off_board_description = "good"
+        else:
+            height_off_board_description = "a bit on the lower side"
+
+    dist_from_board_score = '%.2f' % data['distance_from_board']['raw_score']
+    dist_from_board_frame = Image.fromarray(cv2.cvtColor(frames[data['distance_from_board']['frame_index']], cv2.COLOR_BGR2RGB))
+    dist_buffer = BytesIO()
+    dist_from_board_frame.save(dist_buffer, format='JPEG')
+    encoded_dist_from_board_frame = base64.b64encode(dist_buffer.getvalue()).decode('utf-8')
+    dist_from_board_percentile = data['distance_from_board']['percentile']
+    if 'good' in dist_from_board_percentile:
+        dist_from_board_percentile_status = "Good"
+    elif 'far' in dist_from_board_percentile:
+        dist_from_board_percentile_status = "Too Far"
+    else:
+        dist_from_board_percentile_status = "Too Close"
+
+    knee_bend_score = data['knee_bend']['raw_score']
+    knee_bend_percentile = data['knee_bend']['percentile']
+    knee_bend_frames = []
+    knee_bend_percentile_divided_by_ten = None
+    if knee_bend_score is not None:
+        knee_bend_score = round(float('%.2f' % knee_bend_score))
+        knee_bend_percentile = round(float('%.2f' % (knee_bend_percentile * 100)))
+        knee_bend_frames = frames[data['knee_bend']['frame_indices']]
+        knee_bend_percentile_divided_by_ten = '%.1f' % (data['knee_bend']['percentile'] * 10)
+
+    som_position_tightness_score = data['som_position_tightness']['raw_score']
+    som_position_tightness_percentile = data['som_position_tightness']['percentile']
+    som_position_tightness_position = data['som_position_tightness']['position']
+    som_position_tightness_frames = []
+    som_position_tightness_gif = None
+    som_position_tightness_percentile_divided_by_ten = None
+    if som_position_tightness_score is not None:
+        if is_twister:
+            som_position_tightness_score = round(float('%.2f' % (data['som_position_tightness']['raw_score'] + 15)))
+        else:
+            som_position_tightness_score = round(float('%.2f' % (data['som_position_tightness']['raw_score'])))
+        som_position_tightness_frames = frames[data['som_position_tightness']['frame_indices']]
+        som_position_tightness_gif = generate_gif_from_frames(som_position_tightness_frames)
+        som_position_tightness_percentile = round(float('%.2f' % (som_position_tightness_percentile * 100)))
+        som_position_tightness_percentile_divided_by_ten = '%.1f' % (data['som_position_tightness']['percentile'] * 10)
+
+    twist_position_tightness_score = data['twist_position_tightness']['raw_score']
+    twist_position_tightness_frames = []
+    twist_position_tightness_gif = None
+    twist_position_tightness_percentile = None
+    twist_position_tightness_percentile_divided_by_ten = None
+    if twist_position_tightness_score is not None:
+        twist_position_tightness_score = round(float('%.2f' % twist_position_tightness_score))
+        twist_position_tightness_frames = frames[data['twist_position_tightness']['frame_indices']]
+        twist_position_tightness_gif = generate_gif_from_frames(twist_position_tightness_frames)
+        twist_position_tightness_percentile = round(float('%.2f' % (data['twist_position_tightness']['percentile'] * 100)))
+        twist_position_tightness_percentile_divided_by_ten = '%.1f' % (data['twist_position_tightness']['percentile'] * 10)
+
+    over_under_rotation_score = round(float('%.2f' % data['over_under_rotation']['raw_score']))
+    over_under_rotation_frame = frames[data['over_under_rotation']['frame_index']]
+    over_under_rotation_percentile = round(float('%.2f' % (data['over_under_rotation']['percentile'] * 100)))
+    over_under_rotation_percentile_divided_by_ten = '%.1f' % (data['over_under_rotation']['percentile'] * 10)
+
+    straightness_during_entry_score = round(float('%.2f' % data['straightness_during_entry']['raw_score']))
+    straightness_during_entry_frames = frames[data['straightness_during_entry']['frame_indices']]
+    straightness_during_entry_gif = generate_gif_from_frames(straightness_during_entry_frames, speed_factor = 0.5)
+    straightness_during_entry_percentile = round(float('%.2f' % (data['straightness_during_entry']['percentile'] * 100)))
+    straightness_during_entry_percentile_divided_by_ten = '%.1f' % (data['straightness_during_entry']['percentile'] * 10)
+
+    splash_score = round(float('%.2f' % data['splash']['raw_score']))
+    splash_frame = frames[data['splash']['maximum_index']]
+    splash_indices = frames[data['splash']['frame_indices']]
+    splash_gif = None
+    if len(splash_indices) > 0:
+        splash_gif = generate_gif_from_frames(splash_indices)
+    splash_percentile = round(float('%.2f' % (data['splash']['percentile'] * 100)))
+    splash_percentile_divided_by_ten = '%.1f' % (data['splash']['percentile'] * 10)
+    if float(splash_percentile) < 50:
+        splash_description = 'on the larger side'
+    else:
+        splash_description = 'small'
+    template = env.get_template(template_path)
+    data = {
+        'is_twister' : is_twister,
+        'overall_score_desc' : overall_score_desc,
+        'overall_score' : overall_score,
+        'feet_apart_score' : feet_apart_score,
+        'feet_apart_peaks' : feet_apart_peaks,
+        'has_feet_apart_peaks' : has_feet_apart_peaks,
+        'feet_apart_gif' : feet_apart_gif,
+        'feet_apart_percentile' : feet_apart_percentile,
+        'feet_apart_percentile_divided_by_ten': feet_apart_percentile_divided_by_ten,
+        'include_height_off_platform': include_height_off_platform,
+        'height_off_board_score' : height_off_board_score,
+        'height_off_board_percentile' : height_off_board_percentile,
+        'encoded_height_off_board_frame' : encoded_height_off_board_frame,
+        'height_off_board_percentile_divided_by_ten': height_off_board_percentile_divided_by_ten,
+        'height_off_board_description' : height_off_board_description,
+        'dist_from_board_score' : dist_from_board_score,
+        'dist_from_board_frame' : dist_from_board_frame,
+        'encoded_dist_from_board_frame': encoded_dist_from_board_frame,
+        'dist_from_board_percentile' : dist_from_board_percentile,
+        'dist_from_board_percentile_status': dist_from_board_percentile_status,
+        'knee_bend_score' : knee_bend_score,
+        'knee_bend_frames' : knee_bend_frames,
+        'knee_bend_percentile' : knee_bend_percentile,
+        'knee_bend_percentile_divided_by_ten' : knee_bend_percentile_divided_by_ten,
+        'som_position_tightness_score' : som_position_tightness_score,
+        'som_position_tightness_frames' : som_position_tightness_frames,
+        'som_position_tightness_gif' : som_position_tightness_gif,
+        'som_position_tightness_position' : som_position_tightness_position,
+        'som_position_tightness_percentile' : som_position_tightness_percentile,
+        'som_position_tightness_percentile_divided_by_ten' : som_position_tightness_percentile_divided_by_ten,
+        'twist_position_tightness_score' : twist_position_tightness_score,
+        'twist_position_tightness_frames': twist_position_tightness_frames,
+        'twist_position_tightness_percentile' : twist_position_tightness_percentile,
+        'twist_position_tightness_percentile_divided_by_ten' : twist_position_tightness_percentile_divided_by_ten,
+        'twist_position_tightness_gif' : twist_position_tightness_gif,
+        'over_under_rotation_score' : over_under_rotation_score,
+        'over_under_rotation_frame' : over_under_rotation_frame,
+        'over_under_rotation_percentile' : over_under_rotation_percentile,
+        'over_under_rotation_percentile_divided_by_ten' : over_under_rotation_percentile_divided_by_ten,
+        'straightness_during_entry_score' : straightness_during_entry_score,
+        'straightness_during_entry_gif' : straightness_during_entry_gif,
+        'straightness_during_entry_percentile' : straightness_during_entry_percentile,
+        'straightness_during_entry_percentile_divided_by_ten': straightness_during_entry_percentile_divided_by_ten,
+        'splash_score' : splash_score,
+        'splash_frame' : splash_frame,
+        'splash_gif' : splash_gif,
+        'splash_percentile' : splash_percentile,
+        'splash_percentile_divided_by_ten': splash_percentile_divided_by_ten,
+        'splash_description' : splash_description,
+    }
+    # Render the template with the provided data
+    report_content = template.render(data)
+    return report_content
+
+def generate_symbols_report(template_path, dive_data, frames):
+    # Load the template environment
+    env = Environment(loader=FileSystemLoader('.'))
+    template = env.get_template(template_path)
+    pose_frames = []
+    for i in range(len(dive_data['pose_pred'])):
+        # if dive_data['pose_pred'][i] is not None:
+        pose_frame = draw_symbols(frames[i], dive_data['pose_pred'][i],  dive_data['board_end_coords'][i], dive_data['plat_outputs'][i], dive_data['splash_pred_masks'][i])
+        pose_frames.append(pose_frame)
+    pose_gif = generate_gif_from_frames(pose_frames, speed_factor=2)
+    # image_buffer = BytesIO()
+    # pose_frame.save(image_buffer, format='JPEG')
+    # encoded_pose_frame = base64.b64encode(image_buffer.getvalue()).decode('utf-8')
+    pose_data = {}
+    pose_data['pose_gif'] = pose_gif
+    html = template.render(pose_data)
+    return html
+
+def generate_symbols_report_precomputed(template_path, dive_data, local_directory, progress=gr.Progress()):
+    # Load the template environment
+    file_names = os.listdir(local_directory)
+    file_names.sort()
+    file_names = np.array(file_names)
+
+    if 'above_boards' in dive_data:
+        above_boards = dive_data['above_boards']
+    else:
+        above_boards = [None] * len(file_names)
+
+    env = Environment(loader=FileSystemLoader('.'))
+    template = env.get_template(template_path)
+    pose_frames = []
+    counter = 0
+    for i in range(len(file_names)):
+        progress(i/(len(file_names)+10), desc="Abstracting Symbols")
+        if file_names[i][-4:] != ".jpg":
+            continue
+        # if dive_data['pose_pred'][i] is not None:
+        opencv_image = cv2.imread(local_directory+file_names[i])
+        pose_frame = draw_symbols(opencv_image, dive_data['pose_pred'][counter],  dive_data['board_end_coords'][counter], dive_data['plat_outputs'][counter], dive_data['splash_pred_masks'][counter], above_board=above_boards[counter])
+        pose_frames.append(pose_frame)
+        counter +=1
+    pose_gif = generate_gif_from_frames(pose_frames, speed_factor=2, progress=progress)
+    # image_buffer = BytesIO()
+    # pose_frame.save(image_buffer, format='JPEG')
+    # encoded_pose_frame = base64.b64encode(image_buffer.getvalue()).decode('utf-8')
+    pose_data = {}
+    pose_data['pose_gif'] = pose_gif
+    html = template.render(pose_data)
+    return html

report_template_tables.html

@@ -0,0 +1,149 @@
+<!DOCTYPE html>
+<html>
+    <br>
+    <style>
+        table, th, td {
+          border:1px solid black;
+        }
+    </style>
+    <!-- <br> -->
+    <table style="width:100%">
+        <tr>
+            <td>Your dive was {{overall_score_desc}}, and scored a <strong>{{overall_score}}</strong>.  Here is how we scored each component of the dive. Each percentile is relative to dives in the semifinals and finals of Olympic and World Championship competitions.</td>
+        </tr>
+    </table>
+    <table style="width:100%">
+        <tr>
+            <th style="width:10%">Error</th>
+            <th style="width:70%">Description</th>
+            <th style="width:60%">Visuals</th>
+            <th style="width:10%">Score</th>
+        </tr>
+        <tr>
+            <td>Feet Apart</td>
+            <td>We found that your leg separation angle was on average {{feet_apart_score}}° for your dive. 
+                This is rated as {{feet_apart_percentile}} percentile.</td>
+            <td>
+                {% if has_feet_apart_peaks %} 
+                    <!-- gif  -->
+                    <img src="data:image/gif;base64,{{feet_apart_gif}}" alt="Feet Apart GIF" width="400" height="200">
+                {%else%}
+                    There were no particular instances to show where your feet came apart.
+                {%endif%}
+            </td>
+            <td>
+                {{feet_apart_percentile_divided_by_ten}}
+            </td>
+        </tr>
+        {% if include_height_off_platform %}
+        <tr>
+            <td>Height off platform</td>
+            <td>Your jump was {{height_off_board_description}}, and was rated as {{height_off_board_percentile}} percentile. Here is the highest you jumped off the platform.</td>
+            <td>
+                <img src='data:image/jpg;base64, {{encoded_height_off_board_frame}}' width="400" height="200">
+            </td>
+            <td>
+                {{height_off_board_percentile_divided_by_ten}}
+            </td>
+        </tr>
+        {% endif %}
+        <tr>
+            <td>Distance from platform</td>
+            <td>You were {{dist_from_board_percentile}} the platform. Here is where you came closest to the platform.</td>
+            <td>
+                <!-- <img src='{{local_directory}}/{{dist_from_board_frame}}' width="200" height="100"> -->
+                <img src='data:image/jpg;base64, {{encoded_dist_from_board_frame}}' width="400" height="200">
+            </td>
+            <td>
+                {{dist_from_board_percentile_status}}
+            </td>
+        </tr>
+        {% if som_position_tightness_frames|length > 0 %}
+        <tr>
+            <td>Somersault tightness</td>
+            <td>
+                We found that the tightness of your {{som_position_tightness_position}} was {{som_position_tightness_score}}° on average. 
+                This is rated as {{som_position_tightness_percentile}} percentile. Here are some examples of your position in the somersault.
+            </td>
+            {% if knee_bend_frames|length > 0 %}
+            <td rowspan="2">
+                <!-- gif  -->
+                <img src="data:image/gif;base64,{{som_position_tightness_gif}}" alt="Somersault GIF" width="400" height="200">
+            </td>
+            {% else %}
+            <td>
+                <!-- gif  -->
+                <img src="data:image/gif;base64,{{som_position_tightness_gif}}" alt="Somersault GIF" width="400" height="200">
+            </td>
+            {% endif %}
+            <td>
+                {{som_position_tightness_percentile_divided_by_ten}}
+            </td>
+        </tr>
+        {% endif %}
+        {% if knee_bend_frames|length > 0 %}
+        <tr>
+            <td>Knee straightness</td>
+            <td>
+                We found that your knees bent {{knee_bend_score}}° on average.
+                This is rated as {{knee_bend_percentile}} percentile.
+            </td>
+            <td>
+                {{knee_bend_percentile_divided_by_ten}}
+            </td>
+        </tr>
+        {% endif %}
+        {% if is_twister %}
+        <tr>
+            <td>
+                Twist Straightness
+            </td>
+            <td>
+                We found that the tightness of your {{twist_position_tightness_position}} was {{twist_position_tightness_score}}° on average. 
+                This is rated as {{twist_position_tightness_percentile}} percentile. Here are some examples of your position in the somersault.
+            </td>
+            <td>
+                <!-- gif  -->
+                <img src="data:image/gif;base64,{{twist_position_tightness_gif}}" alt="Twist GIF" width="400" height="200">
+            </td>
+            <td>
+                {{twist_position_tightness_percentile_divided_by_ten}}
+            </td>
+        </tr>
+        {% endif %}
+        <tr>
+            <td>
+                Verticalness (over/under rotation)
+            </td>
+            <td>
+                We found that you deviated from vertical by {{over_under_rotation_score}}°, which was the {{over_under_rotation_percentile}} percentile. 
+            </td>
+            <td rowspan="2">
+                <!-- gif  -->
+                <img src="data:image/gif;base64,{{straightness_during_entry_gif}}" alt="Entry GIF" width="400" height="200">
+            </td>
+            <td>
+                {{over_under_rotation_percentile_divided_by_ten}}
+            </td>
+        </tr>
+        <tr>
+            <td>Body straightness during entry</td>
+            <td>The straightness of your body during entry deviated by {{straightness_during_entry_score}}°, which was the {{straightness_during_entry_percentile}} percentile. </td>
+            <td>{{straightness_during_entry_percentile_divided_by_ten}}</td>
+        </tr>
+        <tr>
+            <td>Splash</td>
+            <td>Your splash was {{splash_description}} and rated as {{splash_percentile}} percentile. </td>
+            <td>
+                <!-- gif  -->
+                <img src="data:image/gif;base64,{{splash_gif}}" alt="Splash GIF" width="400" height="200">
+            </td>
+            <td>
+                {{splash_percentile_divided_by_ten}}
+            </td>
+        </tr>
+
+    </table>
+</html>
+
+

scoring_functions.py

@@ -0,0 +1,579 @@
+import pickle
+import numpy as np
+import os
+import math
+from scipy.signal import find_peaks
+from dive_recognition_functions import *
+
+###  All functions (excluding helper functions) take "dive_data" as input, which is the dictionary with all the information outputted by getAllErrorsAndSegmentation() in tempSegAndAllErrorsForAllFrames.py ###
+
+############## HELPER FUNCTIONS ######################################
+def rotation_direction(vector1, vector2, threshold=0.4):
+    # Calculate the determinant to determine rotation direction
+    determinant = vector1[0] * vector2[1] - vector1[1] * vector2[0]
+    mag1= np.linalg.norm(vector1)
+    mag2= np.linalg.norm(vector2)
+    norm_det = determinant/(mag1*mag2)
+    # print("determinant", determinant/(mag1*mag2))
+    # print(norm_det)
+    if norm_det > threshold:
+        # return "counterclockwise"
+        return 1
+    elif norm_det < 0-threshold:
+        # return "clockwise"
+        return -1
+    else:
+        # return "not determinent"
+        return 0
+
+def find_angle(vector1, vector2):
+    unit_vector_1 = vector1 / np.linalg.norm(vector1)
+    unit_vector_2 = vector2 / np.linalg.norm(vector2)
+    dot_product = np.dot(unit_vector_1, unit_vector_2)
+    angle = math.degrees(np.arccos(dot_product))
+    return angle
+
+#################################################################
+
+def height_off_board_score(dive_data):
+    above_board_indices = [i for i in range(0, len(dive_data['distance_from_board'])) if dive_data['above_boards'][i]==1]
+    takeoff_indices = [i for i in range(0, len(dive_data['takeoff'])) if dive_data['takeoff'][i]==1]
+    final_indices = []
+    prev_board_end_coord = None
+    for i in range(len(above_board_indices)):
+        board_end_coord = dive_data['board_end_coords'][i]
+        if board_end_coord is not None and board_end_coord[1] < 30:
+            continue
+        if board_end_coord is not None and prev_board_end_coord is not None and math.dist(board_end_coord, prev_board_end_coord) > 150:
+            continue
+        if above_board_indices[i] not in takeoff_indices:
+            final_indices.append(above_board_indices[i])
+        prev_board_end_coord = board_end_coord
+    
+    heights = []
+    for i in range(len(final_indices)):
+        pose_pred = dive_data['pose_pred'][final_indices[i]]
+        board_end_coord = dive_data['board_end_coords'][final_indices[i]]
+        if pose_pred is None or board_end_coord is None:
+            continue
+        pose_pred = pose_pred[0]
+        min_height = float('inf')
+        for j in range(len(pose_pred)):
+            if board_end_coord[1] - pose_pred[j][1]< min_height:
+                min_height = board_end_coord[1] - pose_pred[j][1] 
+                if min_height < 0:
+                    min_height = 0
+        heights.append(min_height)
+    if len(heights) == 0:
+        return None, None
+    max_scaled_height = max(heights) / get_scale_factor(dive_data)
+    return max_scaled_height, final_indices[np.argmax(heights)]
+
+def distance_from_board_score(dive_data):
+    above_board_indices = [i for i in range(0, len(dive_data['distance_from_board'])) if dive_data['above_boards'][i]==1]
+    takeoff_indices = [i for i in range(0, len(dive_data['takeoff'])) if dive_data['takeoff'][i]==1]
+    final_indices = []
+    for i in range(len(above_board_indices)):
+        if above_board_indices[i] not in takeoff_indices:
+            final_indices.append(above_board_indices[i])
+    dists = np.array(dive_data['distance_from_board'])[final_indices]
+    for i in range(len(dists)):
+        if dists[i] is None:
+            dists[i] = float('inf')
+    # if np.min(dists) > 75:
+    #     print(dive_data)
+    min_scaled_dist = np.min(dists) / get_scale_factor(dive_data)
+    too_close_threshold = 0.25
+    # if ground_truth[dive_data][0][:2] == '52':
+    #     too_far_threshold = 1.1
+    # too_far_threshold is 75%tile
+    if 'diveNum' in dive_data:
+        if dive_data['diveNum'][0] == '4':
+            too_far_threshold = 1.1
+        if dive_data['diveNum'][0] == '1':
+            too_far_threshold = 1.6
+        if dive_data['diveNum'][0] == '2':
+            too_far_threshold = 1.8
+        if dive_data['diveNum'][0] == '3':
+            too_far_threshold = 1.6
+        if dive_data['diveNum'][0] == '5':
+            too_far_threshold = 1.5
+        if dive_data['diveNum'][0] == '6':
+            too_far_threshold = 1.1
+    else:
+        too_far_threshold = 1.8
+    # good distance
+    if min_scaled_dist < too_far_threshold and min_scaled_dist > too_close_threshold:
+        return 0, min_scaled_dist, final_indices[np.argmin(dists)]
+    # too far
+    if min_scaled_dist >= too_far_threshold:
+        return 1, min_scaled_dist, final_indices[np.argmin(dists)]
+    # too close
+    if min_scaled_dist <= too_close_threshold:
+        return -1, min_scaled_dist, final_indices[np.argmin(dists)]
+    return min_scaled_dist
+
+def knee_bend_score(dive_data):
+    if find_position(dive_data) == 'tuck':
+        return None, None
+    knee_bends = []
+    for i in range(len(dive_data['pose_pred'])):
+        if dive_data['som'][i] == 0:
+            continue
+        pose_pred = dive_data['pose_pred'][i]
+        if pose_pred is None:
+            continue
+        pose_pred = pose_pred[0]
+        knee_to_ankle = [pose_pred[1][0] - pose_pred[0][0], 0-(pose_pred[1][1]-pose_pred[0][1])]
+        knee_to_hip = [pose_pred[1][0] - pose_pred[2][0], 0-(pose_pred[1][1]-pose_pred[2][1])]
+        knee_bend = find_angle(knee_to_ankle, knee_to_hip)
+        knee_bends.append(knee_bend)
+    if len(knee_bends) == 0:
+        return None, None
+    som_indices = [i for i in range(0, len(dive_data['som'])) if dive_data['som'][i]==1]
+    som_avg_knee_bend = np.mean(knee_bends)
+    return 180 - som_avg_knee_bend, som_indices
+
+def position_tightness_score(dive_data):
+    som_indices = [i for i in range(0, len(dive_data['som'])) if dive_data['som'][i]==1]
+    twist_indices = [i for i in range(0, len(dive_data['som'])) if dive_data['twist'][i]==1]
+    som_tightness = np.array(dive_data['position_tightness'])[som_indices]
+    twist_tightness = 180 - np.array(dive_data['position_tightness'])[twist_indices]
+
+    # plt.plot(range(len(som_tightness)), som_tightness)
+    # plt.plot(range(len(twist_tightness)), twist_tightness)
+    # Compute the area using the composite trapezoidal rule.
+    som_tightness = np.array(list(filter(lambda item: item is not None and item < 90, som_tightness)))
+    twist_tightness = np.array(list(filter(lambda item: item is not None and item < 90, twist_tightness)))
+    if len(som_indices) == 0:
+        som_avg = None
+    else:
+        # som_area = np.trapz(som_tightness, dx=5) / len(som_tightness)
+        som_avg = np.mean(som_tightness)
+    # print("som area =", som_area)
+    if len(twist_indices)==0:
+        return som_avg, None, som_indices, twist_indices
+    # twist_area = np.trapz(twist_tightness, dx=5) / len(twist_tightness)
+    twist_avg = np.mean(twist_tightness)
+    if som_avg is not None:
+        som_avg -= 15
+    # print("twist area =",twist_area) 
+    return som_avg, twist_avg, som_indices, twist_indices
+
+def is_rotating_clockwise(dive_data):
+    directions = []
+    for i in range(1, len(dive_data['pose_pred'])):
+        if dive_data['pose_pred'][i] is None or dive_data['pose_pred'][i-1] is None:
+            continue
+        if dive_data['on_boards'][i] == 0:
+            prev_pose_pred_hip = dive_data['pose_pred'][i-1][0][3]
+            curr_pose_pred_hip = dive_data['pose_pred'][i][0][3]
+            prev_pose_pred_knee = dive_data['pose_pred'][i-1][0][4]
+            curr_pose_pred_knee = dive_data['pose_pred'][i][0][4]   
+            prev_hip_knee = [prev_pose_pred_knee[0] - prev_pose_pred_hip[0], 0-(prev_pose_pred_knee[1] - prev_pose_pred_hip[1])]
+            curr_hip_knee = [curr_pose_pred_knee[0] - curr_pose_pred_hip[0], 0-(curr_pose_pred_knee[1] - curr_pose_pred_hip[1])]
+            direction = rotation_direction(prev_hip_knee, curr_hip_knee, threshold=0)
+            directions.append(direction)
+    return np.sum(directions) < 0
+
+def over_under_rotation_score(dive_data):
+    entry_indices = [i for i in range(0, len(dive_data['entry'])) if dive_data['entry'][i]==1]
+    over_under_rotation_error = np.array(dive_data['over_under_rotation'])[entry_indices]
+    splashes = np.array(dive_data['splash'])[entry_indices]
+    for i in range(len(over_under_rotation_error) - 1, -1, -1):
+        if over_under_rotation_error[i] is None:
+            continue
+        else:
+            # gets the second to last pose (assuming the last pose has incorrect pose estimation)
+            # print(over_under_rotation_error[i-1]-90)
+            index = i-2
+            if index < 0:
+                index = 0
+            total_index = entry_indices[index]
+            if splashes[index] is None and over_under_rotation_error[index] is not None:
+                # print(entry_indices)
+                # print(entry_indices[i-3])
+                # return np.cos(math.radians(over_under_rotation_error[i-1]-90+10))
+                # avg_leg_torso = over_under_rotation_error[i-1]-90 +10 - straightness_during_entry_score(dive_data)
+                pose_pred = dive_data['pose_pred'][total_index][0]
+                thorax_pelvis_vector = [pose_pred[1][0] - pose_pred[7][0], 0-(pose_pred[1][1]-pose_pred[7][1])]
+                prev_pose_pred = dive_data['pose_pred'][total_index - 1]
+                if prev_pose_pred is not None:
+                    # print(dive_data)
+                    prev_pose_pred = prev_pose_pred[0]
+                    prev_thorax_pelvis_vector = [prev_pose_pred[1][0] - prev_pose_pred[7][0], 0-(prev_pose_pred[1][1]-prev_pose_pred[7][1])]
+                    rotation_speed = find_angle(thorax_pelvis_vector, prev_thorax_pelvis_vector)
+                else:
+                    rotation_speed = 10
+                # print(rotation_speed)
+                vector2 = [0, 1]
+                # print(find_angle(thorax_pelvis_vector, vector2))
+                # print(thorax_pelvis_vector, prev_thorax_pelvis_vector)
+                if is_rotating_clockwise(dive_data):
+                    # if under-rotated
+                    if thorax_pelvis_vector[0] < 0:
+                        avg_leg_torso = find_angle(thorax_pelvis_vector, vector2) - rotation_speed
+                        
+                    else:
+                        avg_leg_torso = find_angle(thorax_pelvis_vector, vector2) + rotation_speed
+                else:
+                    # if over-rotated
+                    if thorax_pelvis_vector[0] < 0:  
+                        avg_leg_torso = find_angle(thorax_pelvis_vector, vector2) + rotation_speed
+                    else:
+                        avg_leg_torso = find_angle(thorax_pelvis_vector, vector2) - rotation_speed
+                # avg_leg_torso = over_under_rotation_error[i-1]-90 +10 - np.array(dive_data['position_tightness'])[entry_indices][i-1]
+                return np.abs(avg_leg_torso), entry_indices[index]
+                break
+
+def straightness_during_entry_score(dive_data):
+    entry_indices = [i for i in range(0, len(dive_data['entry'])) if dive_data['entry'][i]==1]
+    straightness_during_entry = np.array(dive_data['position_tightness'])[entry_indices]
+    over_under_rotation = over_under_rotation_score(dive_data)
+    if over_under_rotation is not None:
+        frame = over_under_rotation[1]
+        index = entry_indices.index(frame) - 1
+        if index < 0:
+            index = 0
+        return 180-straightness_during_entry[index], [frame-1, frame, frame + 1]
+    splashes = np.array(dive_data['splash'])[entry_indices]
+    for i in range(len(straightness_during_entry) - 1, -1, -1):
+        if i > 0 and (straightness_during_entry[i] is None or splashes[i] is not None):
+            continue
+        else:
+            # gets the second to last pose (assuming the last pose has incorrect pose estimation)
+            if straightness_during_entry[i] is not None:
+                if 180-straightness_during_entry[i] > 130:
+                    continue
+                return 180-straightness_during_entry[i], entry_indices[i-1:i+2]
+                break
+
+def splash_score(dive_data):
+    entry_indices = [i for i in range(0, len(dive_data['entry'])) if dive_data['entry'][i]==1]
+    if len(entry_indices) == 0:
+        return None
+    splash_indices=[i for i in range(0, len(dive_data['splash'])) if dive_data['splash'][i] is not None]
+    splashes = np.array(dive_data['splash'])[entry_indices]
+    for i in range(len(splashes)):
+        if splashes[i] is None:
+            splashes[i] = 0
+    splashes = splashes / get_scale_factor(dive_data)**2
+
+    # area under curve
+    # plt.plot(range(len(splashes)), splashes)
+    area = np.trapz(splashes, dx=5) 
+    # print("area =", area)
+    
+    # if area < 0.002:
+    #     print(dive_data)
+    return area, entry_indices[np.argmax(splashes)], splash_indices
+
+# feet apart
+def feet_apart_score(dive_data):
+    takeoff_indices = [i for i in range(0, len(dive_data['takeoff'])) if dive_data['takeoff'][i]==1]
+    non_takeoff_indices = [i for i in range(len(dive_data['takeoff'])) if (i not in takeoff_indices and dive_data['splash'][i] is None)]
+    feet_apart_error = np.array(dive_data['feet_apart'])[non_takeoff_indices]
+    # plt.plot(range(len(feet_apart_error)), feet_apart_error)
+    for i in range(len(feet_apart_error)):
+        if feet_apart_error[i] is None or math.isnan(feet_apart_error[i]):
+            feet_apart_error[i] = 0
+    peaks, _ = find_peaks(feet_apart_error, height=5)
+    if len(peaks) >= 1:
+        peak_indices = np.array(non_takeoff_indices)[peaks]
+    # elif len(peaks) == 1:
+    #     peak_indices = non_takeoff_indices[peaks[0]]
+    else:
+        peak_indices = []
+    # print(len(peaks))
+    # return peaks
+    # area = np.trapz(feet_apart_error, dx=5) /len(non_takeoff_indices)
+    area = np.mean(feet_apart_error)
+
+    # print(area)
+    return area, peak_indices
+
+def find_position(dive_data):
+    angles = []
+    three_in_a_row = 0
+    for i in range(1, len(dive_data['pose_pred'])):
+        pose_pred = dive_data['pose_pred'][i]
+        if pose_pred is None or dive_data['som'][i]==0:
+            continue
+        pose_pred = pose_pred[0]
+        l_knee = pose_pred[4]
+        l_ankle = pose_pred[5]
+        l_hip = pose_pred[3]
+        l_knee_ankle = [l_ankle[0] - l_knee[0], 0-(l_ankle[1] - l_knee[1])]
+        l_knee_hip = [l_hip[0] - l_knee[0], 0-(l_hip[1] - l_knee[1])]
+        angle = find_angle(l_knee_ankle, l_knee_hip)
+        angles.append(angle)
+        # print(angle)
+        if angle < 70:
+            three_in_a_row += 1
+            if three_in_a_row >=3:
+                return 'tuck'
+        else:
+            three_in_a_row =0
+    # if np.mean(angles) < 100:
+    #     return 'tuck'
+    # print(som_counter(dive_data))
+    # print(twist_counter(dive_data))
+    # if twist_counter_full_dive(dive_data) > 0 and som_counter_full_dive(dive_data)[0] < 5:
+    #     return 'free'
+    return 'pike'
+
+def get_position_from_diveNum(dive_data):
+    diveNum = dive_data['diveNum']
+    position_code = diveNum[-1]
+    if position_code == 'a':
+        return "straight"
+    elif position_code == 'b':
+        return "pike"
+    elif position_code == 'c':
+        return "tuck"
+    elif position_code == 'd':
+        return "free"
+    else:
+        return None
+
+def get_all_report_scores(dive_data):
+    # with open('all_distributions.npy', 'rb') as f:
+    #     feet_apart_scores = np.load(f)
+    #     distance_from_board_scores = np.load(f)
+    #     som_position_tightness_scores = np.load(f)
+    #     twist_position_tightness_scores = np.load(f)
+    #     over_under_rotation_scores = np.load(f)
+    #     straightness_during_entry_scores = np.load(f)
+    #     splash_scores = np.load(f)
+    with open('distribution_data.pkl', 'rb') as f:
+        distribution_data = pickle.load(f)
+     ## handstand and som_count##
+    expected_som, handstand = som_counter_full_dive(dive_data)
+    ## twist_count
+    expected_twists = twist_counter_full_dive(dive_data)
+    ## direction: front, back, reverse, inward
+    expected_direction = get_direction(dive_data)
+    dive_data['is_handstand'] = handstand
+    dive_data['direction'] = expected_direction
+
+    intermediate_scores = {}
+    all_percentiles = []
+    entry_indices = [i for i in range(0, len(dive_data['entry'])) if dive_data['entry'][i]==1]
+
+    ### height off board ###
+    if dive_data['is_handstand']:
+        error_scores = distribution_data['armstand_height_off_board_scores']
+    elif expected_twists >0:
+        error_scores = distribution_data['twist_height_off_board_scores']
+    elif dive_data['direction']=='front':
+        error_scores = distribution_data['front_height_off_board_scores']
+    elif dive_data['direction']=='back':
+        error_scores = distribution_data['back_height_off_board_scores']
+    elif dive_data['direction']=='reverse':
+        error_scores = distribution_data['reverse_height_off_board_scores']
+    elif dive_data['direction']=='inward':
+        error_scores = distribution_data['inward_height_off_board_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['height_off_board'] = {}
+    if dive_data['is_handstand']:
+        intermediate_scores['height_off_board']['raw_score'] = None
+        intermediate_scores['height_off_board']['frame_index'] = None
+    else:
+        intermediate_scores['height_off_board']['raw_score'] = height_off_board_score(dive_data)[0]
+        intermediate_scores['height_off_board']['frame_index'] = height_off_board_score(dive_data)[1]
+    err = intermediate_scores['height_off_board']['raw_score']
+    if err is not None:
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['height_off_board']['percentile'] = temp.index(err)/len(temp)
+        all_percentiles.append(temp.index(err)/len(temp))
+    else:
+        intermediate_scores['height_off_board']['percentile'] = None
+
+    ## distance from board ####
+    error_scores = distribution_data['distance_from_board_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['distance_from_board'] = {}
+    intermediate_scores['distance_from_board']['raw_score'] = distance_from_board_score(dive_data)[1]
+    intermediate_scores['distance_from_board']['frame_index'] = distance_from_board_score(dive_data)[2]
+    err = distance_from_board_score(dive_data)[0]
+    if err is not None:
+        if err == 1:     
+            intermediate_scores['distance_from_board']['percentile'] = "safe, but too far from"
+            intermediate_scores['distance_from_board']['score'] = 0.5
+
+        elif err == 0:
+            intermediate_scores['distance_from_board']['percentile'] = "a good distance from"
+            intermediate_scores['distance_from_board']['score'] = 1
+        else:
+            intermediate_scores['distance_from_board']['percentile'] = "too close to"
+            intermediate_scores['distance_from_board']['score'] = 0
+        all_percentiles.append(intermediate_scores['distance_from_board']['score'])
+    else:
+        intermediate_scores['distance_from_board']['percentile'] = None
+        intermediate_scores['distance_from_board']['score'] = None
+        
+    ### feet_apart_scores ###
+    error_scores = distribution_data['feet_apart_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['feet_apart'] = {}
+    intermediate_scores['feet_apart']['raw_score'] = feet_apart_score(dive_data)[0]
+    intermediate_scores['feet_apart']['peaks'] = feet_apart_score(dive_data)[1]
+    err = intermediate_scores['feet_apart']['raw_score']
+    if err is not None:
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['feet_apart']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+    else:
+        intermediate_scores['feet_apart']['percentile'] = None
+
+    ### knee_bend_scores ###
+    error_scores = distribution_data['knee_bend_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['knee_bend'] = {}
+    intermediate_scores['knee_bend']['raw_score'] = knee_bend_score(dive_data)[0]
+    intermediate_scores['knee_bend']['frame_indices'] = knee_bend_score(dive_data)[1]
+    err = intermediate_scores['knee_bend']['raw_score']
+    if err is not None:
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['knee_bend']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+    else:
+        intermediate_scores['knee_bend']['percentile'] = None
+        
+
+    ### som_position_tightness_scores ###
+    error_scores = distribution_data['som_position_tightness_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['som_position_tightness'] = {}
+    position = find_position(dive_data)
+    if position == 'tuck':
+        intermediate_scores['som_position_tightness']['position'] = 'tuck'
+    else:
+        intermediate_scores['som_position_tightness']['position'] = 'pike'
+    intermediate_scores['som_position_tightness']['raw_score'] = position_tightness_score(dive_data)[0]
+    intermediate_scores['som_position_tightness']['frame_indices'] = position_tightness_score(dive_data)[2]
+    err = intermediate_scores['som_position_tightness']['raw_score']
+    if err is not None:
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['som_position_tightness']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+    else:
+        intermediate_scores['som_position_tightness']['percentile'] = None
+    
+    ### twist_position_tightness_scores ###
+    error_scores = distribution_data['twist_position_tightness_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['twist_position_tightness'] = {}
+    intermediate_scores['twist_position_tightness']['raw_score'] = position_tightness_score(dive_data)[1]
+    intermediate_scores['twist_position_tightness']['frame_indices'] = position_tightness_score(dive_data)[3]
+    err = intermediate_scores['twist_position_tightness']['raw_score']
+    if err is not None:
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['twist_position_tightness']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+    else:
+        intermediate_scores['twist_position_tightness']['percentile'] = None
+        
+    ### over_under_rotation_scores ###
+    error_scores = distribution_data['over_under_rotation_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['over_under_rotation'] = {}
+    if over_under_rotation_score(dive_data) is not None:
+        intermediate_scores['over_under_rotation']['raw_score'] = over_under_rotation_score(dive_data)[0]
+        intermediate_scores['over_under_rotation']['frame_index'] = over_under_rotation_score(dive_data)[1]
+    else:
+        intermediate_scores['over_under_rotation']['raw_score'] = None
+        intermediate_scores['over_under_rotation']['frame_index'] = None
+    err = intermediate_scores['over_under_rotation']['raw_score']
+    if err is not None:
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['over_under_rotation']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+
+    else:
+        intermediate_scores['over_under_rotation']['percentile'] = None
+        
+    ### splash_scores ###
+    error_scores = distribution_data['splash_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['splash'] = {}
+    intermediate_scores['splash']['raw_score'] = splash_score(dive_data)[0]
+    intermediate_scores['splash']['maximum_index'] = splash_score(dive_data)[1]
+    intermediate_scores['splash']['frame_indices'] = splash_score(dive_data)[2]
+
+    err = intermediate_scores['splash']['raw_score']
+    if err is not None:    
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['splash']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+
+    else:
+        intermediate_scores['splash']['percentile'] = None
+        
+    ### straightness_during_entry_scores ###
+    error_scores = distribution_data['straightness_during_entry_scores']
+    error_scores = list(filter(lambda item: item is not None, error_scores))
+    intermediate_scores['straightness_during_entry'] = {}
+    if straightness_during_entry_score(dive_data) is not None:
+        intermediate_scores['straightness_during_entry']['raw_score'] = straightness_during_entry_score(dive_data)[0]
+        intermediate_scores['straightness_during_entry']['frame_indices'] = straightness_during_entry_score(dive_data)[1]
+    else:
+        intermediate_scores['straightness_during_entry']['raw_score'] = None
+        intermediate_scores['straightness_during_entry']['frame_index'] = None
+        
+    err = intermediate_scores['straightness_during_entry']['raw_score']
+    if err is not None:    
+        temp = error_scores
+        temp.append(err)
+        temp.sort()
+        intermediate_scores['straightness_during_entry']['percentile'] = 1-temp.index(err)/len(temp)
+        all_percentiles.append(1-temp.index(err)/len(temp))
+
+    else:
+        intermediate_scores['straightness_during_entry']['percentile'] = None
+        
+        ## overall score ###
+        # Excellent:                   10 
+        # Very Good:             8.5-9.5 
+        # Good:                      7.0-8.0 
+        # Satisfactory:           5.0-6.5 
+        # Deficient:                2.5-4.5 
+        # Unsatisfactory:   	0.5-2.0 
+        # Completely failed:  	0  
+    overall_score = np.mean(all_percentiles) * 10
+    intermediate_scores['overall_score'] = {}
+    intermediate_scores['overall_score']['raw_score'] = overall_score
+    if overall_score == 10:
+        intermediate_scores['overall_score']['description'] = 'excellent'
+    elif overall_score >=8.5 and overall_score <10:
+        intermediate_scores['overall_score']['description'] = 'very good'
+    elif overall_score >=7 and overall_score <8.5:
+        intermediate_scores['overall_score']['description'] = 'good'
+    elif overall_score >=5 and overall_score <7:
+        intermediate_scores['overall_score']['description'] = 'satisfactory'
+    elif overall_score >=2.5 and overall_score <5:
+        intermediate_scores['overall_score']['description'] = 'deficient'
+    elif overall_score >0 and overall_score <2.5:
+        intermediate_scores['overall_score']['description'] = 'unsatisfactory'
+    else:
+        intermediate_scores['overall_score']['description'] = 'completely failed'
+    
+    return intermediate_scores
+
+
+
+

tempSegAndAllErrorsForAllFrames.py

@@ -0,0 +1,201 @@
+from microprograms.temporal_segmentation.entry import entry_microprogram_one_frame
+from microprograms.temporal_segmentation.somersault import somersault_microprogram_one_frame
+from microprograms.temporal_segmentation.twist import twist_microprogram_one_frame
+from microprograms.temporal_segmentation.start_takeoff import takeoff_microprogram_one_frame
+from microprograms.errors.distance_from_springboard_micro_program import board_end
+from microprograms.errors.splash_micro_program import get_splash_from_one_frame
+from microprograms.errors.distance_from_springboard_micro_program import calculate_distance_from_springboard_for_one_frame
+from microprograms.errors.distance_from_springboard_micro_program import calculate_distance_from_platform_for_one_frame
+from microprograms.errors.angles_micro_programs import applyFeetApartError
+from microprograms.errors.angles_micro_programs import applyPositionTightnessError
+from models.detectron2.platform_detector_setup import get_platform_detector
+from models.pose_estimator.pose_estimator_model_setup import get_pose_estimation
+from models.detectron2.diver_detector_setup import get_diver_detector
+from models.pose_estimator.pose_estimator_model_setup import get_pose_model
+from models.detectron2.splash_detector_setup import get_splash_detector
+from somersault_counter import som_counter, twist_counter
+from microprograms.errors.over_rotation import over_rotation
+from temporal_segmentation import detect_on_board
+import pickle
+import os
+import math
+import numpy as np
+import cv2
+with open('segmentation_error_data.pkl', 'rb') as f:
+    data = pickle.load(f)
+
+def getDiveInfo_from_diveNum(diveNum):
+    handstand = (diveNum[0] == '6')
+    expected_som = int(diveNum[2])
+    if len(diveNum) == 5:
+        expected_twists = int(diveNum[3])
+    else:
+        expected_twists = 0
+    if diveNum[0] == '1' or diveNum[0] == '3' or diveNum[:2] == '51' or diveNum[:2] == '53' or diveNum[:2] == '61' or diveNum[:2] == '63':
+        back_facing = False
+    else:
+        back_facing = True
+    if diveNum[0] == '1' or diveNum[:2] == '51' or diveNum[:2] == '61':
+        expected_direction = 'front'
+    elif diveNum[0] == '2' or diveNum[:2] == '52' or diveNum[:2] == '62':
+        expected_direction = 'back'
+    elif diveNum[0] == '3' or diveNum[:2] == '53' or diveNum[:2] == '63':
+        expected_direction = 'reverse'
+    elif diveNum[0] == '4':
+        expected_direction = 'inward'
+    if diveNum[-1] == 'b':
+        position = 'pike'
+    elif diveNum[-1] == 'c':
+        position = 'tuck'
+    else:
+        position = 'free'
+    return handstand, expected_som, expected_twists, back_facing, expected_direction, position
+
+def getAllErrorsAndSegmentation(first_folder, second_folder, diveNum, board_side=None, platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None):
+    handstand, expected_som, expected_twists, back_facing, expected_direction, position = getDiveInfo_from_diveNum(diveNum)
+    # first_folder = input("what is the first folder? Ex: 01, FINAWorldChampionships2019_Women10m_final_r1, etc. ")
+    # second_folder = input("what is the second folder? (dive within the first folder)")
+    dive_data = {}
+    takeoff = []
+    twist = []
+    som = []
+    entry = []
+    distance_from_board = []
+    position_tightness = []
+    feet_apart = []
+    over_under_rotation = []
+    splash = []
+    pose_preds = []
+    diver_boxes = []
+    above_boards = []
+    on_boards = []
+    som_counts = []
+    twist_counts = []
+    board_end_coords = []
+    plat_outputs = []
+    splash_pred_masks = []
+    above_board = True
+    on_board = True
+    if platform_detector is None:
+        platform_detector = get_platform_detector()
+    if splash_detector is None:
+        splash_detector = get_splash_detector()
+    if diver_detector is None:
+        diver_detector = get_diver_detector()
+    if pose_model is None:
+        pose_model = get_pose_model()
+    key = (first_folder, int(second_folder))
+    dive_folder_num = "{}_{}".format(first_folder, second_folder)
+    directory = './FineDiving/datasets/FINADiving_MTL_256s/{}/{}/'.format(first_folder, second_folder)
+    file_names = os.listdir(directory)
+    # with open('./output/joint_plots/{}/pose_preds.pkl'.format(dive_folder_num), 'rb') as pickle_file:
+    #     pose_preds = pickle.load(pickle_file)
+    j = 0
+    prev_pred = None
+    som_prev_pred = None
+    half_som_count=0
+    petal_count = 0
+    in_petal = False
+    for i in range(len(file_names)):
+        # pose_pred = None
+        filepath = directory + file_names[i]
+        # print("filepath:", filepath)
+        if file_names[i][-4:] != ".jpg":
+            continue
+        diver_box, pose_pred = get_pose_estimation(filepath, diver_detector=diver_detector, pose_model=pose_model)
+        # pose_pred = data[key]['pose_pred'][i]
+        diver_boxes.append(diver_box)
+        pose_preds.append(pose_pred)
+        # if j < len(pose_preds):
+        #     print("filepath has pose_pred:", filepath)
+        #     pose_pred = pose_preds[j]
+        #     j += 1
+        calculated_half_som_count, skip = som_counter(pose_pred, prev_pose_pred=som_prev_pred, half_som_count=half_som_count, handstand=handstand)
+        if not skip:
+            som_prev_pred = pose_pred
+        calculated_petal_count, calculated_in_petal = twist_counter(pose_pred, prev_pose_pred=prev_pred, in_petal=in_petal, petal_count=petal_count)
+        im = cv2.imread(filepath)
+        plat_output = platform_detector(im)
+        plat_outputs.append(plat_output)
+        board_end_coord = board_end(plat_output, board_side=board_side)
+        board_end_coords.append(board_end_coord)
+        # if board_end_coord is None:
+        #     print("NO BOARD NONE CRYING")
+        if above_board and not on_board and board_end_coord is not None and pose_pred is not None and np.array(pose_pred)[0][2][1] > int(board_end_coord[1]):
+            above_board=False
+        if on_board and detect_on_board(board_end_coord, board_side, pose_pred, handstand) is not None and not detect_on_board(board_end_coord, board_side, pose_pred, handstand):
+            on_board = False
+        if above_board:
+            above_boards.append(1)
+        else:
+            above_boards.append(0)
+        if on_board:
+            on_boards.append(1)
+        else:
+            on_boards.append(0)
+        calculated_takeoff = takeoff_microprogram_one_frame(filepath, above_board=above_board, on_board=on_board, pose_pred=pose_pred)
+        calculated_twist = twist_microprogram_one_frame(filepath, on_board=on_board, pose_pred=pose_pred, expected_twists=expected_twists, petal_count=petal_count, expected_som=expected_som, half_som_count=half_som_count, diver_detector=diver_detector, pose_model=pose_model)
+        calculated_som = somersault_microprogram_one_frame(filepath, pose_pred=pose_pred, on_board=on_board, expected_som=expected_som, half_som_count=half_som_count, expected_twists=expected_twists, petal_count=petal_count, diver_detector=diver_detector, pose_model=pose_model)
+        calculated_entry = entry_microprogram_one_frame(filepath, above_board=above_board, on_board=on_board, pose_pred=pose_pred, expected_twists=expected_twists, petal_count=petal_count, expected_som=expected_som, half_som_count=half_som_count, splash_detector=splash_detector, visualize=False, dive_folder_num=dive_folder_num)
+        if calculated_som == 1:
+            half_som_count = calculated_half_som_count
+        elif calculated_twist == 1:
+            half_som_count = calculated_half_som_count
+            petal_count = calculated_petal_count
+            in_petal = calculated_in_petal
+        # distance from board
+        dist = calculate_distance_from_platform_for_one_frame(filepath, visualize=False, pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model, board_end_coord=board_end_coord, platform_detector=platform_detector) # saves photo to ./output/data/distance_from_board/
+        distance_from_board.append(dist)
+        position_tightness.append(applyPositionTightnessError(filepath, pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+        feet_apart.append(applyFeetApartError(filepath, pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+        over_under_rotation.append(over_rotation(filepath, pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+        splash_area, splash_pred_mask = get_splash_from_one_frame(filepath, predictor=splash_detector, visualize=False)
+        splash.append(splash_area)
+        splash_pred_masks.append(splash_pred_mask)    
+        takeoff.append(calculated_takeoff)
+        twist.append(calculated_twist)
+        som.append(calculated_som)
+        entry.append(calculated_entry)
+        som_counts.append(half_som_count)
+        twist_counts.append(petal_count)
+        prev_pred = pose_pred
+
+    dive_data['pose_pred'] = pose_preds
+    dive_data['takeoff'] = takeoff
+    dive_data['twist'] = twist
+    dive_data['som'] = som
+    dive_data['entry'] = entry
+    dive_data['distance_from_board'] = distance_from_board
+    dive_data['position_tightness'] = position_tightness
+    dive_data['feet_apart'] = feet_apart
+    dive_data['over_under_rotation'] = over_under_rotation
+    dive_data['splash'] = splash
+    dive_data['above_boards'] = above_boards
+    dive_data['on_boards'] = on_boards
+    dive_data['som_counts'] = som_counts
+    dive_data['twist_counts'] = twist_counts
+    dive_data['board_end_coords'] = board_end_coords
+    dive_data['diver_boxes'] = diver_boxes
+    dive_data['splash_pred_masks'] = splash_pred_masks
+    dive_data['plat_outputs'] = plat_outputs
+    dive_data['board_side'] = board_side
+    dive_data['is_handstand'] = handstand
+    dive_data['direction'] = expected_direction
+
+    print("takeoff", takeoff)
+    print("twist", twist)
+    print("som", som)
+    print("entry", entry)
+    print("distance_from_board", distance_from_board)
+    print("position_tightness", position_tightness)
+    print("feet_apart", feet_apart)
+    print("over_under_rotation", over_under_rotation)
+    print("splash", splash)
+    print("above_boards", above_boards)
+    print("on_boards", on_boards)
+    print("som_counts", som_counts)
+    print("twist_counts", twist_counts)
+    print("board_end_coords", board_end_coords)
+    print("diver_boxes", diver_boxes)
+    return dive_data
+

tempSegAndAllErrorsForAllFrames_newVids.py

@@ -0,0 +1,312 @@
+from microprograms.temporal_segmentation.entry import entry_microprogram_one_frame
+from microprograms.temporal_segmentation.somersault import somersault_microprogram_one_frame
+from microprograms.temporal_segmentation.twist import twist_microprogram_one_frame
+from microprograms.temporal_segmentation.start_takeoff import takeoff_microprogram_one_frame
+from microprograms.errors.distance_from_springboard_micro_program import board_end
+from microprograms.errors.splash_micro_program import *
+from microprograms.errors.distance_from_springboard_micro_program import calculate_distance_from_springboard_for_one_frame
+from microprograms.errors.distance_from_springboard_micro_program import calculate_distance_from_platform_for_one_frame
+from microprograms.errors.distance_from_springboard_micro_program import find_which_side_board_on
+from microprograms.errors.angles_micro_programs import applyFeetApartError
+from microprograms.errors.angles_micro_programs import applyPositionTightnessError
+from models.detectron2.platform_detector_setup import get_platform_detector
+from models.pose_estimator.pose_estimator_model_setup import get_pose_estimation
+from models.detectron2.diver_detector_setup import get_diver_detector
+from models.pose_estimator.pose_estimator_model_setup import get_pose_model
+from models.detectron2.splash_detector_setup import get_splash_detector
+from somersault_counter import som_counter, twist_counter
+from microprograms.errors.over_rotation import over_rotation
+from temporal_segmentation import detect_on_board
+from dive_recognition_functions import *
+from scoring_functions import get_scale_factor
+import gradio as gr
+import pickle
+import os
+import math
+import numpy as np
+import cv2
+
+with open('segmentation_error_data.pkl', 'rb') as f:
+    data = pickle.load(f)
+
+def getDiveInfo_from_diveNum(diveNum):
+    handstand = (diveNum[0] == '6')
+    expected_som = int(diveNum[2])
+    if len(diveNum) == 5:
+        expected_twists = int(diveNum[3])
+    else:
+        expected_twists = 0
+    if diveNum[0] == '1' or diveNum[0] == '3' or diveNum[:2] == '51' or diveNum[:2] == '53' or diveNum[:2] == '61' or diveNum[:2] == '63':
+        back_facing = False
+    else:
+        back_facing = True
+    if diveNum[0] == '1' or diveNum[:2] == '51' or diveNum[:2] == '61':
+        expected_direction = 'front'
+    elif diveNum[0] == '2' or diveNum[:2] == '52' or diveNum[:2] == '62':
+        expected_direction = 'back'
+    elif diveNum[0] == '3' or diveNum[:2] == '53' or diveNum[:2] == '63':
+        expected_direction = 'reverse'
+    elif diveNum[0] == '4':
+        expected_direction = 'inward'
+    if diveNum[-1] == 'b':
+        position = 'pike'
+    elif diveNum[-1] == 'c':
+        position = 'tuck'
+    else:
+        position = 'free'
+    return handstand, expected_som, expected_twists, back_facing, expected_direction, position
+
+def getDiveInfo_from_symbols(frames, dive_data=None, platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None):
+    print("Getting dive info from symbols...")
+    if dive_data is None:
+        print("somethings not getting passed in properly")
+        dive_data = abstractSymbols(frames, platform_detector=platform_detector, splash_detector=splash_detector, diver_detector=diver_detector, pose_model=pose_model)
+
+    # get above_boards, on_boards, and position_tightness
+    above_board = True
+    on_board = True
+    above_boards = []
+    on_boards = []
+    position_tightness = []
+    distances = []
+    prev_board_coord = None
+    for i in range(len(dive_data['pose_pred'])):
+        pose_pred = dive_data['pose_pred'][i]
+        board_end_coord =  dive_data['board_end_coords'][i]
+        if board_end_coord is not None and prev_board_coord is not None:
+            distances.append(math.dist(board_end_coord, prev_board_coord))
+            if math.dist(board_end_coord, prev_board_coord) > 150:
+                position_tightness.append(applyPositionTightnessError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+                if above_board:
+                    above_boards.append(1)
+                else:
+                    above_boards.append(0)
+                if on_board:
+                    on_boards.append(1)
+                else:
+                    on_boards.append(0)
+                continue
+        if above_board and not on_board and board_end_coord is not None and pose_pred is not None and np.array(pose_pred)[0][2][1] > int(board_end_coord[1]):
+            above_board=False
+        if on_board:
+            handstand = is_handstand(dive_data)
+            calculate_on_board = detect_on_board(board_end_coord, dive_data['board_side'], pose_pred, handstand)
+            if calculate_on_board is not None and not calculate_on_board:
+                on_board = False
+        if above_board:
+            above_boards.append(1)
+        else:
+            above_boards.append(0)
+        if on_board:
+            on_boards.append(1)
+        else:
+            on_boards.append(0)
+        prev_board_coord = board_end_coord
+        position_tightness.append(applyPositionTightnessError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+    dive_data['on_boards'] = on_boards
+    dive_data['above_boards'] = above_boards
+    dive_data['position_tightness'] = position_tightness
+
+    ## handstand and som_count##
+    expected_som, handstand = som_counter_full_dive(dive_data)
+
+    ## twist_count
+    expected_twists = twist_counter_full_dive(dive_data)
+
+    ## direction: front, back, reverse, inward
+    expected_direction = get_direction(dive_data)
+
+    return handstand, expected_som, expected_twists, expected_direction, dive_data
+
+
+def abstractSymbols(frames, progress=gr.Progress(), platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None):
+    print("Abstracting symbols...")
+    splashes = []
+    pose_preds = []
+    board_sides = []
+    plat_outputs = []
+    diver_boxes = []
+    splash_pred_masks = []
+    if platform_detector is None:
+        platform_detector = get_platform_detector()
+    if splash_detector is None:
+        splash_detector = get_splash_detector()
+    if diver_detector is None:
+        diver_detector = get_diver_detector()
+    if pose_model is None:
+        pose_model = get_pose_model()
+    num_frames = len(frames)
+    i = 0
+    for frame in frames:
+        progress(i/num_frames, desc="Abstracting Symbols")
+        plat_output = platform_detector(frame)
+        plat_outputs.append(plat_output)
+        board_side = find_which_side_board_on(plat_output)
+        if board_side is not None:
+            board_sides.append(board_side)
+        diver_box, pose_pred = get_pose_estimation(filepath="", image_bgr=frame, diver_detector=diver_detector, pose_model=pose_model)
+        pose_preds.append(pose_pred)
+        diver_boxes.append(diver_box)
+        splash_area, splash_pred_mask = get_splash_from_one_frame(filepath="", im=frame, predictor=splash_detector, visualize=False)
+        splash_pred_masks.append(splash_pred_mask)
+        splashes.append(splash_area)
+        i+=1
+    dive_data = {}
+    dive_data['plat_outputs'] = plat_outputs
+    dive_data['pose_pred'] = pose_preds
+    dive_data['splash'] = splashes
+    dive_data['splash_pred_masks'] = splash_pred_masks
+    dive_data['board_sides'] = board_sides
+    board_sides.sort()
+    board_side = board_sides[len(board_sides)//2]
+    dive_data['board_side'] = board_side
+    dive_data['diver_boxes'] = diver_boxes
+
+     # get board_end_coords
+    board_end_coords = []
+    for plat_output in dive_data['plat_outputs']:
+        board_end_coord = board_end(plat_output, board_side=dive_data['board_side'])
+        board_end_coords.append(board_end_coord)
+    dive_data['board_end_coords'] = board_end_coords
+
+    return dive_data
+
+def getAllErrorsAndSegmentation_newVids(frames, dive_data, progress=gr.Progress(), diveNum="", board_side=None, platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None):
+    print("in getAllErrorsAndSegmentation function...")
+    if len(frames) != len(dive_data['pose_pred']):
+        raise gr.Error("Abstract Symbols first!")
+    if diveNum != "":
+        dive_num_given = True
+        handstand, expected_som, expected_twists, back_facing, expected_direction, position = getDiveInfo_from_diveNum(diveNum)
+    else:
+        dive_num_given = False
+        handstand, expected_som, expected_twists, expected_direction, dive_data = getDiveInfo_from_symbols(frames, dive_data=dive_data, platform_detector=platform_detector, splash_detector=splash_detector, diver_detector=diver_detector, pose_model=pose_model)
+
+    if not dive_num_given:
+        above_boards = dive_data['above_boards']
+        on_boards = dive_data['on_boards']
+        position_tightness = dive_data['position_tightness']
+        board_end_coords = dive_data['board_end_coords']
+    else:
+        above_board = True
+        on_board = True
+        above_boards = []
+        on_boards = []
+        board_end_coords = []
+        position_tightness = []
+    splash = dive_data['splash'] 
+    diver_boxes = dive_data['diver_boxes']   
+    board_side = dive_data['board_side']
+    pose_preds = dive_data['pose_pred']
+    takeoff = []
+    twist = []
+    som = []
+    entry = []
+    distance_from_board = []
+    feet_apart = []
+    over_under_rotation = []
+    som_counts = []
+    twist_counts = []
+    
+    if platform_detector is None:
+        platform_detector = get_platform_detector()
+    if splash_detector is None:
+        splash_detector = get_splash_detector()
+    if diver_detector is None:
+        diver_detector = get_diver_detector()
+    if pose_model is None:
+        pose_model = get_pose_model()
+    j = 0
+    prev_pred = None
+    som_prev_pred = None
+    half_som_count=0
+    petal_count = 0
+    in_petal = False
+    num_frames = len(frames)
+    for i in range(num_frames):
+        progress(i/num_frames, desc="Calculating Dive Errors")
+        pose_pred = pose_preds[i]
+        calculated_half_som_count, skip = som_counter(pose_pred, prev_pose_pred=som_prev_pred, half_som_count=half_som_count, handstand=handstand)
+        if not skip:
+            som_prev_pred = pose_pred
+        calculated_petal_count, calculated_in_petal = twist_counter(pose_pred, prev_pose_pred=prev_pred, in_petal=in_petal, petal_count=petal_count)
+        if dive_num_given:
+            outputs = platform_detector(frames[i])
+            board_end_coord = board_end(outputs, board_side=board_side)
+            board_end_coords.append(board_end_coord)
+            if above_board and not on_board and board_end_coord is not None and pose_pred is not None and np.array(pose_pred)[0][2][1] > int(board_end_coord[1]):
+                above_board=False
+            if on_board and detect_on_board(board_end_coord, board_side, pose_pred, handstand) is not None and not detect_on_board(board_end_coord, board_side, pose_pred, handstand):
+                on_board = False
+            if above_board:
+                above_boards.append(1)
+            else:
+                above_boards.append(0)
+            if on_board:
+                on_boards.append(1)
+            else:
+                on_boards.append(0)
+        else:
+            board_end_coord = board_end_coords[i]
+            above_board = (above_boards[i] == 1)
+            on_board = (on_boards[i] == 1)
+        calculated_takeoff = takeoff_microprogram_one_frame(filepath="", above_board=above_board, on_board=on_board, pose_pred=pose_pred)
+        calculated_twist = twist_microprogram_one_frame(filepath="", on_board=on_board, pose_pred=pose_pred, expected_twists=expected_twists, petal_count=petal_count, expected_som=expected_som, half_som_count=half_som_count, diver_detector=diver_detector, pose_model=pose_model)
+        calculated_som = somersault_microprogram_one_frame(filepath="", pose_pred=pose_pred, on_board=on_board, expected_som=expected_som, half_som_count=half_som_count, expected_twists=expected_twists, petal_count=petal_count, diver_detector=diver_detector, pose_model=pose_model)
+        calculated_entry = entry_microprogram_one_frame(filepath="", frame=frames[i], above_board=above_board, on_board=on_board, pose_pred=pose_pred, expected_twists=expected_twists, petal_count=petal_count, expected_som=expected_som, half_som_count=half_som_count, splash_detector=splash_detector, visualize=False)
+        if calculated_som == 1:
+            half_som_count = calculated_half_som_count
+        elif calculated_twist == 1:
+            half_som_count = calculated_half_som_count
+            petal_count = calculated_petal_count
+            in_petal = calculated_in_petal
+        # distance from board
+        dist = calculate_distance_from_platform_for_one_frame(filepath="", im=frames[i], visualize=False, pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model, board_end_coord=board_end_coord, platform_detector=platform_detector) # saves photo to ./output/data/distance_from_board/
+        distance_from_board.append(dist)
+        if dive_num_given:
+            position_tightness.append(applyPositionTightnessError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+            # splash.append(get_splash_from_one_frame(filepath="", im=frames[i], predictor=splash_detector, visualize=False))
+        feet_apart.append(applyFeetApartError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+        over_under_rotation.append(over_rotation(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model))
+        takeoff.append(calculated_takeoff)
+        twist.append(calculated_twist)
+        som.append(calculated_som)
+        entry.append(calculated_entry)
+        som_counts.append(half_som_count)
+        twist_counts.append(petal_count)
+        prev_pred = pose_pred
+    print("takeoff", takeoff)
+    print("twist", twist)
+    print("som", som)
+    print("entry", entry)
+    print("distance_from_board", distance_from_board)
+    print("position_tightness", position_tightness)
+    print("feet_apart", feet_apart)
+    print("over_under_rotation", over_under_rotation)
+    print("splash", splash)
+    print("above_boards", above_boards)
+    print("on_boards", on_boards)
+    print("som_counts", som_counts)
+    print("twist_counts", twist_counts)
+    print("board_end_coords", board_end_coords)
+    print("diver_boxes", diver_boxes)
+
+    print("saving data into dive_data dictionary...")
+    dive_data['takeoff'] = takeoff
+    dive_data['twist'] = twist
+    dive_data['som'] = som
+    dive_data['entry'] = entry
+    dive_data['distance_from_board'] = distance_from_board
+    dive_data['position_tightness'] = position_tightness
+    dive_data['feet_apart'] = feet_apart
+    dive_data['over_under_rotation'] = over_under_rotation
+    dive_data['above_boards'] = above_boards
+    dive_data['on_boards'] = on_boards
+    dive_data['som_counts'] = som_counts
+    dive_data['twist_counts'] = twist_counts
+    dive_data['board_end_coords'] = board_end_coords
+    dive_data['is_handstand'] = handstand
+    dive_data['direction'] = expected_direction
+    return dive_data
+
+