from microprograms.temporal_segmentation.entry import entry_microprogram_one_frame from microprograms.temporal_segmentation.somersault import somersault_microprogram_one_frame from microprograms.temporal_segmentation.twist import twist_microprogram_one_frame from microprograms.temporal_segmentation.start_takeoff import takeoff_microprogram_one_frame from microprograms.errors.distance_from_springboard_micro_program import board_end from microprograms.errors.splash_micro_program import * from microprograms.errors.distance_from_springboard_micro_program import calculate_distance_from_springboard_for_one_frame from microprograms.errors.distance_from_springboard_micro_program import calculate_distance_from_platform_for_one_frame from microprograms.errors.distance_from_springboard_micro_program import find_which_side_board_on from microprograms.errors.angles_micro_programs import applyFeetApartError from microprograms.errors.angles_micro_programs import applyPositionTightnessError from models.detectron2.platform_detector_setup import get_platform_detector from models.pose_estimator.pose_estimator_model_setup import get_pose_estimation from models.detectron2.diver_detector_setup import get_diver_detector from models.pose_estimator.pose_estimator_model_setup import get_pose_model from models.detectron2.splash_detector_setup import get_splash_detector from somersault_counter import som_counter, twist_counter from microprograms.errors.over_rotation import over_rotation from temporal_segmentation import detect_on_board from dive_recognition_functions import * from scoring_functions import get_scale_factor import gradio as gr import pickle import os import math import numpy as np import cv2 with open('segmentation_error_data.pkl', 'rb') as f: data = pickle.load(f) def getDiveInfo_from_diveNum(diveNum): handstand = (diveNum[0] == '6') expected_som = int(diveNum[2]) if len(diveNum) == 5: expected_twists = int(diveNum[3]) else: expected_twists = 0 if diveNum[0] == '1' or diveNum[0] == '3' or diveNum[:2] == '51' or diveNum[:2] == '53' or diveNum[:2] == '61' or diveNum[:2] == '63': back_facing = False else: back_facing = True if diveNum[0] == '1' or diveNum[:2] == '51' or diveNum[:2] == '61': expected_direction = 'front' elif diveNum[0] == '2' or diveNum[:2] == '52' or diveNum[:2] == '62': expected_direction = 'back' elif diveNum[0] == '3' or diveNum[:2] == '53' or diveNum[:2] == '63': expected_direction = 'reverse' elif diveNum[0] == '4': expected_direction = 'inward' if diveNum[-1] == 'b': position = 'pike' elif diveNum[-1] == 'c': position = 'tuck' else: position = 'free' return handstand, expected_som, expected_twists, back_facing, expected_direction, position def getDiveInfo_from_symbols(frames, dive_data=None, platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None): print("Getting dive info from symbols...") if dive_data is None: print("somethings not getting passed in properly") dive_data = abstractSymbols(frames, platform_detector=platform_detector, splash_detector=splash_detector, diver_detector=diver_detector, pose_model=pose_model) # get above_boards, on_boards, and position_tightness above_board = True on_board = True above_boards = [] on_boards = [] position_tightness = [] distances = [] prev_board_coord = None for i in range(len(dive_data['pose_pred'])): pose_pred = dive_data['pose_pred'][i] board_end_coord = dive_data['board_end_coords'][i] if board_end_coord is not None and prev_board_coord is not None: distances.append(math.dist(board_end_coord, prev_board_coord)) if math.dist(board_end_coord, prev_board_coord) > 150: position_tightness.append(applyPositionTightnessError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model)) if above_board: above_boards.append(1) else: above_boards.append(0) if on_board: on_boards.append(1) else: on_boards.append(0) continue if above_board and not on_board and board_end_coord is not None and pose_pred is not None and np.array(pose_pred)[0][2][1] > int(board_end_coord[1]): above_board=False if on_board: handstand = is_handstand(dive_data) calculate_on_board = detect_on_board(board_end_coord, dive_data['board_side'], pose_pred, handstand) if calculate_on_board is not None and not calculate_on_board: on_board = False if above_board: above_boards.append(1) else: above_boards.append(0) if on_board: on_boards.append(1) else: on_boards.append(0) prev_board_coord = board_end_coord position_tightness.append(applyPositionTightnessError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model)) dive_data['on_boards'] = on_boards dive_data['above_boards'] = above_boards dive_data['position_tightness'] = position_tightness ## handstand and som_count## expected_som, handstand = som_counter_full_dive(dive_data) ## twist_count expected_twists = twist_counter_full_dive(dive_data) ## direction: front, back, reverse, inward expected_direction = get_direction(dive_data) return handstand, expected_som, expected_twists, expected_direction, dive_data def abstractSymbols(frames, progress=gr.Progress(), platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None): print("Abstracting symbols...") splashes = [] pose_preds = [] board_sides = [] plat_outputs = [] diver_boxes = [] splash_pred_masks = [] if platform_detector is None: platform_detector = get_platform_detector() if splash_detector is None: splash_detector = get_splash_detector() if diver_detector is None: diver_detector = get_diver_detector() if pose_model is None: pose_model = get_pose_model() num_frames = len(frames) i = 0 for frame in frames: progress(i/num_frames, desc="Abstracting Symbols") plat_output = platform_detector(frame) plat_outputs.append(plat_output) board_side = find_which_side_board_on(plat_output) if board_side is not None: board_sides.append(board_side) diver_box, pose_pred = get_pose_estimation(filepath="", image_bgr=frame, diver_detector=diver_detector, pose_model=pose_model) pose_preds.append(pose_pred) diver_boxes.append(diver_box) splash_area, splash_pred_mask = get_splash_from_one_frame(filepath="", im=frame, predictor=splash_detector, visualize=False) splash_pred_masks.append(splash_pred_mask) splashes.append(splash_area) i+=1 dive_data = {} dive_data['plat_outputs'] = plat_outputs dive_data['pose_pred'] = pose_preds dive_data['splash'] = splashes dive_data['splash_pred_masks'] = splash_pred_masks dive_data['board_sides'] = board_sides board_sides.sort() board_side = board_sides[len(board_sides)//2] dive_data['board_side'] = board_side dive_data['diver_boxes'] = diver_boxes # get board_end_coords board_end_coords = [] for plat_output in dive_data['plat_outputs']: board_end_coord = board_end(plat_output, board_side=dive_data['board_side']) board_end_coords.append(board_end_coord) dive_data['board_end_coords'] = board_end_coords return dive_data def getAllErrorsAndSegmentation_newVids(frames, dive_data, progress=gr.Progress(), diveNum="", board_side=None, platform_detector=None, splash_detector=None, diver_detector=None, pose_model=None): print("in getAllErrorsAndSegmentation function...") if len(frames) != len(dive_data['pose_pred']): raise gr.Error("Abstract Symbols first!") if diveNum != "": dive_num_given = True handstand, expected_som, expected_twists, back_facing, expected_direction, position = getDiveInfo_from_diveNum(diveNum) else: dive_num_given = False handstand, expected_som, expected_twists, expected_direction, dive_data = getDiveInfo_from_symbols(frames, dive_data=dive_data, platform_detector=platform_detector, splash_detector=splash_detector, diver_detector=diver_detector, pose_model=pose_model) if not dive_num_given: above_boards = dive_data['above_boards'] on_boards = dive_data['on_boards'] position_tightness = dive_data['position_tightness'] board_end_coords = dive_data['board_end_coords'] else: above_board = True on_board = True above_boards = [] on_boards = [] board_end_coords = [] position_tightness = [] splash = dive_data['splash'] diver_boxes = dive_data['diver_boxes'] board_side = dive_data['board_side'] pose_preds = dive_data['pose_pred'] takeoff = [] twist = [] som = [] entry = [] distance_from_board = [] feet_apart = [] over_under_rotation = [] som_counts = [] twist_counts = [] if platform_detector is None: platform_detector = get_platform_detector() if splash_detector is None: splash_detector = get_splash_detector() if diver_detector is None: diver_detector = get_diver_detector() if pose_model is None: pose_model = get_pose_model() j = 0 prev_pred = None som_prev_pred = None half_som_count=0 petal_count = 0 in_petal = False num_frames = len(frames) for i in range(num_frames): progress(i/num_frames, desc="Calculating Dive Errors") pose_pred = pose_preds[i] calculated_half_som_count, skip = som_counter(pose_pred, prev_pose_pred=som_prev_pred, half_som_count=half_som_count, handstand=handstand) if not skip: som_prev_pred = pose_pred calculated_petal_count, calculated_in_petal = twist_counter(pose_pred, prev_pose_pred=prev_pred, in_petal=in_petal, petal_count=petal_count) if dive_num_given: outputs = platform_detector(frames[i]) board_end_coord = board_end(outputs, board_side=board_side) board_end_coords.append(board_end_coord) if above_board and not on_board and board_end_coord is not None and pose_pred is not None and np.array(pose_pred)[0][2][1] > int(board_end_coord[1]): above_board=False if on_board and detect_on_board(board_end_coord, board_side, pose_pred, handstand) is not None and not detect_on_board(board_end_coord, board_side, pose_pred, handstand): on_board = False if above_board: above_boards.append(1) else: above_boards.append(0) if on_board: on_boards.append(1) else: on_boards.append(0) else: board_end_coord = board_end_coords[i] above_board = (above_boards[i] == 1) on_board = (on_boards[i] == 1) calculated_takeoff = takeoff_microprogram_one_frame(filepath="", above_board=above_board, on_board=on_board, pose_pred=pose_pred) calculated_twist = twist_microprogram_one_frame(filepath="", on_board=on_board, pose_pred=pose_pred, expected_twists=expected_twists, petal_count=petal_count, expected_som=expected_som, half_som_count=half_som_count, diver_detector=diver_detector, pose_model=pose_model) calculated_som = somersault_microprogram_one_frame(filepath="", pose_pred=pose_pred, on_board=on_board, expected_som=expected_som, half_som_count=half_som_count, expected_twists=expected_twists, petal_count=petal_count, diver_detector=diver_detector, pose_model=pose_model) calculated_entry = entry_microprogram_one_frame(filepath="", frame=frames[i], above_board=above_board, on_board=on_board, pose_pred=pose_pred, expected_twists=expected_twists, petal_count=petal_count, expected_som=expected_som, half_som_count=half_som_count, splash_detector=splash_detector, visualize=False) if calculated_som == 1: half_som_count = calculated_half_som_count elif calculated_twist == 1: half_som_count = calculated_half_som_count petal_count = calculated_petal_count in_petal = calculated_in_petal # distance from board dist = calculate_distance_from_platform_for_one_frame(filepath="", im=frames[i], visualize=False, pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model, board_end_coord=board_end_coord, platform_detector=platform_detector) # saves photo to ./output/data/distance_from_board/ distance_from_board.append(dist) if dive_num_given: position_tightness.append(applyPositionTightnessError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model)) # splash.append(get_splash_from_one_frame(filepath="", im=frames[i], predictor=splash_detector, visualize=False)) feet_apart.append(applyFeetApartError(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model)) over_under_rotation.append(over_rotation(filepath="", pose_pred=pose_pred, diver_detector=diver_detector, pose_model=pose_model)) takeoff.append(calculated_takeoff) twist.append(calculated_twist) som.append(calculated_som) entry.append(calculated_entry) som_counts.append(half_som_count) twist_counts.append(petal_count) prev_pred = pose_pred print("takeoff", takeoff) print("twist", twist) print("som", som) print("entry", entry) print("distance_from_board", distance_from_board) print("position_tightness", position_tightness) print("feet_apart", feet_apart) print("over_under_rotation", over_under_rotation) print("splash", splash) print("above_boards", above_boards) print("on_boards", on_boards) print("som_counts", som_counts) print("twist_counts", twist_counts) print("board_end_coords", board_end_coords) print("diver_boxes", diver_boxes) print("saving data into dive_data dictionary...") dive_data['takeoff'] = takeoff dive_data['twist'] = twist dive_data['som'] = som dive_data['entry'] = entry dive_data['distance_from_board'] = distance_from_board dive_data['position_tightness'] = position_tightness dive_data['feet_apart'] = feet_apart dive_data['over_under_rotation'] = over_under_rotation dive_data['above_boards'] = above_boards dive_data['on_boards'] = on_boards dive_data['som_counts'] = som_counts dive_data['twist_counts'] = twist_counts dive_data['board_end_coords'] = board_end_coords dive_data['is_handstand'] = handstand dive_data['direction'] = expected_direction return dive_data