import onnxruntime as rt
import numpy as np
import json
import torch
import cv2
import os
from torch.utils.data.dataset import Dataset
import random
import math
import argparse
# Constants and paths defining model, image, and dataset specifics
MODEL_DIR = './movenet_int8.onnx' # Path to the MoveNet model
IMG_SIZE = 192 # Image size used for processing
FEATURE_MAP_SIZE = 48 # Feature map size used in the model
CENTER_WEIGHT_ORIGIN_PATH = './center_weight_origin.npy' # Path to center weight origin file
DATASET_PATH = 'your_dataset_path' # Base path for the dataset
EVAL_LABLE_PATH = os.path.join(DATASET_PATH, "val2017.json") # Path to validation labels JSON file
EVAL_IMG_PATH = os.path.join(DATASET_PATH, 'imgs') # Path to validation images
def getDist(pre, labels):
"""
Calculate the Euclidean distance between predicted and labeled keypoints.
Args:
pre: Predicted keypoints [batchsize, 14]
labels: Labeled keypoints [batchsize, 14]
Returns:
dist: Distance between keypoints [batchsize, 7]
"""
pre = pre.reshape([-1, 17, 2])
labels = labels.reshape([-1, 17, 2])
res = np.power(pre[:,:,0]-labels[:,:,0],2)+np.power(pre[:,:,1]-labels[:,:,1],2)
return res
def getAccRight(dist, th = 5/IMG_SIZE):
"""
Compute accuracy for each keypoint based on a threshold.
Args:
dist: Distance between keypoints [batchsize, 7]
th: Threshold for accuracy computation
Returns:
res: Accuracy per keypoint [7,] representing the count of correct predictions
"""
res = np.zeros(dist.shape[1], dtype=np.int64)
for i in range(dist.shape[1]):
res[i] = sum(dist[:,i]
47] = 47
reg_x[reg_x<0] = 0
reg_y[reg_y>47] = 47
reg_y[reg_y<0] = 0
score = heatmaps[dim0,dim1+n,reg_y,reg_x]
offset_x = offsets[dim0,dim1+n*2,reg_y,reg_x]#*img_size//4
offset_y = offsets[dim0,dim1+n*2+1,reg_y,reg_x]#*img_size//4
res_x = (reg_x+offset_x)/(img_size//4)
res_y = (reg_y+offset_y)/(img_size//4)
res_x[score47] = 47
reg_x[reg_x<0] = 0
reg_y[reg_y>47] = 47
reg_y[reg_y<0] = 0
offset_x = offsets[dim0,dim1+n*2,reg_y,reg_x]#*img_size//4
offset_y = offsets[dim0,dim1+n*2+1,reg_y,reg_x]#*img_size//4
res_x = (reg_x+offset_x)/(img_size//4)
res_y = (reg_y+offset_y)/(img_size//4)
res_x[kps_mask[:,n]==0] = -1
res_y[kps_mask[:,n]==0] = -1
res.extend([res_x, res_y])
res = np.concatenate(res,axis=1) #bs*14
return res
def label2heatmap(keypoints, other_keypoints, img_size):
'''
Convert labeled keypoints to heatmaps for keypoints.
Args:
keypoints: Target person's keypoints
other_keypoints: Other people's keypoints
img_size: Size of the image
Returns:
heatmaps: Heatmaps for keypoints
sigma: Value used for heatmap generation
'''
#keypoints: target person
#other_keypoints: other people's keypoints need to be add to the heatmap
heatmaps = []
keypoints_range = np.reshape(keypoints,(-1,3))
keypoints_range = keypoints_range[keypoints_range[:,2]>0]
min_x = np.min(keypoints_range[:,0])
min_y = np.min(keypoints_range[:,1])
max_x = np.max(keypoints_range[:,0])
max_y = np.max(keypoints_range[:,1])
area = (max_y-min_y)*(max_x-min_x)
sigma = 3
if area < 0.16:
sigma = 3
elif area < 0.3:
sigma = 5
else:
sigma = 7
for i in range(0,len(keypoints),3):
if keypoints[i+2]==0:
heatmaps.append(np.zeros((img_size//4, img_size//4)))
continue
x = int(keypoints[i]*img_size//4)
y = int(keypoints[i+1]*img_size//4)
if x==img_size//4:x=(img_size//4-1)
if y==img_size//4:y=(img_size//4-1)
if x>img_size//4 or x<0:x=-1
if y>img_size//4 or y<0:y=-1
heatmap = generate_heatmap(x, y, other_keypoints[i//3], (img_size//4, img_size//4),sigma)
heatmaps.append(heatmap)
heatmaps = np.array(heatmaps, dtype=np.float32)
return heatmaps,sigma
def generate_heatmap(x, y, other_keypoints, size, sigma):
'''
Generate a heatmap for a specific keypoint.
Args:
x, y: Absolute position of the keypoint
other_keypoints: Position of other keypoints
size: Size of the heatmap
sigma: Value used for heatmap generation
Returns:
heatmap: Generated heatmap for the keypoint
'''
#x,y abs postion
#other_keypoints positive position
sigma+=6
heatmap = np.zeros(size)
if x<0 or y<0 or x>=size[0] or y>=size[1]:
return heatmap
tops = [[x,y]]
if len(other_keypoints)>0:
#add other people's keypoints
for i in range(len(other_keypoints)):
x = int(other_keypoints[i][0]*size[0])
y = int(other_keypoints[i][1]*size[1])
if x==size[0]:x=(size[0]-1)
if y==size[1]:y=(size[1]-1)
if x>size[0] or x<0 or y>size[1] or y<0: continue
tops.append([x,y])
for top in tops:
#heatmap[top[1]][top[0]] = 1
x,y = top
x0 = max(0,x-sigma//2)
x1 = min(size[0],x+sigma//2)
y0 = max(0,y-sigma//2)
y1 = min(size[1],y+sigma//2)
for map_y in range(y0, y1):
for map_x in range(x0, x1):
d2 = ((map_x - x) ** 2 + (map_y - y) ** 2)**0.5
if d2<=sigma//2:
heatmap[map_y, map_x] += math.exp(-d2/(sigma//2)*3)
if heatmap[map_y, map_x] > 1:
heatmap[map_y, map_x] = 1
# heatmap[heatmap<0.1] = 0
return heatmap
def label2center(cx, cy, other_centers, img_size, sigma):
'''
Convert labeled keypoints to a center heatmap.
Args:
cx, cy: Center coordinates
other_centers: Other people's centers
img_size: Size of the image
sigma: Value used for heatmap generation
Returns:
heatmaps: Heatmap representing the center
'''
heatmaps = []
heatmap = generate_heatmap(cx, cy, other_centers, (img_size//4, img_size//4),sigma+2)
heatmaps.append(heatmap)
heatmaps = np.array(heatmaps, dtype=np.float32)
return heatmaps
def label2reg(keypoints, cx, cy, img_size):
'''
Convert labeled keypoints to regression maps.
Args:
keypoints: Labeled keypoints
cx, cy: Center coordinates
img_size: Size of the image
Returns:
heatmaps: Regression maps for keypoints
'''
heatmaps = np.zeros((len(keypoints)//3*2, img_size//4, img_size//4), dtype=np.float32)
for i in range(len(keypoints)//3):
if keypoints[i*3+2]==0:
continue
x = keypoints[i*3]*img_size//4
y = keypoints[i*3+1]*img_size//4
if x==img_size//4:x=(img_size//4-1)
if y==img_size//4:y=(img_size//4-1)
if x>img_size//4 or x<0 or y>img_size//4 or y<0:
continue
reg_x = x-cx
reg_y = y-cy
for j in range(cy-2,cy+3):
if j<0 or j>img_size//4-1:
continue
for k in range(cx-2,cx+3):
if k<0 or k>img_size//4-1:
continue
if cximg_size//4 or small_x<0 or small_y>img_size//4 or small_y<0:
continue
heatmaps[i*2][small_y][small_x] = offset_x#/(img_size//4)
heatmaps[i*2+1][small_y][small_x] = offset_y#/(img_size//4)
return heatmaps
class TensorDataset(Dataset):
'''
Custom Dataset class for handling data loading and preprocessing
'''
def __init__(self, data_labels, img_dir, img_size, data_aug=None):
self.data_labels = data_labels
self.img_dir = img_dir
self.data_aug = data_aug
self.img_size = img_size
self.interp_methods = [cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA,
cv2.INTER_NEAREST, cv2.INTER_LANCZOS4]
def __getitem__(self, index):
item = self.data_labels[index]
"""
item = {
"img_name":save_name,
"keypoints":save_keypoints,
"center":save_center,
"other_centers":other_centers,
"other_keypoints":other_keypoints,
}
"""
# [name,h,w,keypoints...]
img_path = os.path.join(self.img_dir, item["img_name"])
img = cv2.imread(img_path, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (self.img_size, self.img_size),
interpolation=random.choice(self.interp_methods))
#### Data Augmentation
if self.data_aug is not None:
img, item = self.data_aug(img, item)
img = img.astype(np.float32)
img = np.transpose(img,axes=[2,0,1])
keypoints = item["keypoints"]
center = item['center']
other_centers = item["other_centers"]
other_keypoints = item["other_keypoints"]
kps_mask = np.ones(len(keypoints)//3)
for i in range(len(keypoints)//3):
if keypoints[i*3+2]==0:
kps_mask[i] = 0
heatmaps,sigma = label2heatmap(keypoints, other_keypoints, self.img_size) #(17, 48, 48)
cx = min(max(0,int(center[0]*self.img_size//4)),self.img_size//4-1)
cy = min(max(0,int(center[1]*self.img_size//4)),self.img_size//4-1)
centers = label2center(cx, cy, other_centers, self.img_size, sigma) #(1, 48, 48)
regs = label2reg(keypoints, cx, cy, self.img_size) #(14, 48, 48)
offsets = label2offset(keypoints, cx, cy, regs, self.img_size)#(14, 48, 48)
labels = np.concatenate([heatmaps,centers,regs,offsets],axis=0)
img = img / 127.5 - 1.0
return img, labels, kps_mask, img_path
def __len__(self):
return len(self.data_labels)
# Function to get data loader based on mode (e.g., evaluation)
def getDataLoader(mode, input_data):
'''
Function to get data loader based on mode (e.g., evaluation).
Args:
mode: Mode of data loader (e.g., 'eval')
input_data: Input data
Returns:
data_loader: DataLoader for specified mode
'''
if mode=="eval":
val_loader = torch.utils.data.DataLoader(
TensorDataset(input_data[0],
EVAL_IMG_PATH,
IMG_SIZE,
),
batch_size=1,
shuffle=False,
num_workers=0,
pin_memory=False)
return val_loader
# Class for managing data and obtaining evaluation data loader
class Data():
'''
Class for managing data and obtaining evaluation data loader.
'''
def __init__(self):
pass
def getEvalDataloader(self):
with open(EVAL_LABLE_PATH, 'r') as f:
data_label_list = json.loads(f.readlines()[0])
print("[INFO] Total images: ", len(data_label_list))
input_data = [data_label_list]
data_loader = getDataLoader("eval",
input_data)
return data_loader
# Configs for onnx inference session
def make_parser():
'''
Create parser for MoveNet ONNX runtime inference.
Returns:
parser: Argument parser for MoveNet inference
'''
parser = argparse.ArgumentParser("movenet onnxruntime inference")
parser.add_argument(
"--ipu",
action="store_true",
help="Use IPU for inference.",
)
parser.add_argument(
"--provider_config",
type=str,
default="vaip_config.json",
help="Path of the config file for seting provider_options.",
)
return parser.parse_args()
if __name__ == '__main__':
args = make_parser()
if args.ipu:
providers = ["VitisAIExecutionProvider"]
provider_options = [{"config_file": args.provider_config}]
else:
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
provider_options = None
# Get evaluation data loader using the Data class
data = Data()
data_loader = data.getEvalDataloader()
# Load MoveNet model using ONNX runtime
model = rt.InferenceSession(MODEL_DIR, providers=providers, provider_options=provider_options)
correct = 0
total = 0
# Loop through the data loader for evaluation
for batch_idx, (imgs, labels, kps_mask, img_names) in enumerate(data_loader):
if batch_idx%100 == 0:
print('Finish ',batch_idx)
imgs = imgs.detach().cpu().numpy()
output = model.run(['1548','1607','1665','1723'],{'blob.1':imgs})
pre = movenetDecode(output, kps_mask,mode='output',img_size=IMG_SIZE)
gt = movenetDecode(labels, kps_mask,mode='label',img_size=IMG_SIZE)
acc = myAcc(pre, gt)
correct += sum(acc)
total += len(acc)
# Compute and print accuracy based on evaluated data
acc = correct/total
print('[Info] acc: {:.3f}% \n'.format(100. * acc)) |