File size: 4,244 Bytes
0ca2a11 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
"""
The code was adapted from the MICCAI FLARE Challenge
https://flare22.grand-challenge.org/
The testing images will be evaluated one by one.
To compensate for the Docker container startup time, we give a time tolerance for the running time.
https://neurips22-cellseg.grand-challenge.org/metrics/
"""
import os
join = os.path.join
import sys
import shutil
import time
import torch
import argparse
from collections import OrderedDict
from skimage import io
import tifffile as tif
import numpy as np
import pandas as pd
parser = argparse.ArgumentParser('Segmentation efficiency eavluation for docker containers', add_help=False)
parser.add_argument('-i', '--test_img_path', default='./val-imgs-30/', type=str, help='testing data path')
parser.add_argument('-o','--save_path', default='./val_team_seg', type=str, help='segmentation output path')
parser.add_argument('-d','--docker_folder_path', default='./team_docker', type=str, help='team docker path')
args = parser.parse_args()
test_img_path = args.test_img_path
save_path = args.save_path
docker_path = args.docker_folder_path
input_temp = './inputs/'
output_temp = './outputs'
os.makedirs(save_path, exist_ok=True)
dockers = sorted(os.listdir(docker_path))
test_cases = sorted(os.listdir(test_img_path))
for docker in dockers:
try:
# create temp folers for inference one-by-one
if os.path.exists(input_temp):
shutil.rmtree(input_temp)
if os.path.exists(output_temp):
shutil.rmtree(output_temp)
os.makedirs(input_temp)
os.makedirs(output_temp)
# load docker and create a new folder to save segmentation results
teamname = docker.split('.')[0].lower()
print('teamname docker: ', docker)
# os.system('docker image load < {}'.format(join(docker_path, docker)))
team_outpath = join(save_path, teamname)
if os.path.exists(team_outpath):
shutil.rmtree(team_outpath)
os.mkdir(team_outpath)
metric = OrderedDict()
metric['Img Name'] = []
metric['Real Running Time'] = []
metric['Rank Running Time'] = []
# To obtain the running time for each case, we inference the testing case one-by-one
for case in test_cases:
shutil.copy(join(test_img_path, case), input_temp)
if case.endswith('.tif') or case.endswith('.tiff'):
img = tif.imread(join(input_temp, case))
else:
img = io.imread(join(input_temp, case))
pix_num = img.shape[0] * img.shape[1]
cmd = 'docker container run --gpus="device=0" -m 28g --name {} --rm -v $PWD/inputs/:/workspace/inputs/ -v $PWD/outputs/:/workspace/outputs/ {}:latest /bin/bash -c "sh predict.sh" '.format(teamname, teamname)
print(teamname, ' docker command:', cmd, '\n', 'testing image name:', case)
start_time = time.time()
os.system(cmd)
real_running_time = time.time() - start_time
print(f"{case} finished! Inference time: {real_running_time}")
# save metrics
metric['Img Name'].append(case)
metric['Real Running Time'].append(real_running_time)
if pix_num <= 1000000:
rank_running_time = np.max([0, real_running_time-10])
else:
rank_running_time = np.max([0, real_running_time-10*(pix_num/1000000)])
metric['Rank Running Time'].append(rank_running_time)
os.remove(join(input_temp, case))
seg_name = case.split('.')[0] + '_label.tiff'
try:
os.rename(join(output_temp, seg_name), join(team_outpath, seg_name))
except:
print(f"{join(output_temp, seg_name)}, {join(team_outpath, seg_name)}")
print("Wrong segmentation name!!! It should be image_name.split(\'.\')[0] + \'_label.tiff\' ")
metric_df = pd.DataFrame(metric)
metric_df.to_csv(join(team_outpath, teamname + '_running_time.csv'), index=False)
torch.cuda.empty_cache()
# os.system("docker rmi {}:latest".format(teamname))
shutil.rmtree(input_temp)
shutil.rmtree(output_temp)
except Exception as e:
print(e)
|