Spaces:
Running
Running
| import glob | |
| import json | |
| import math | |
| import operator | |
| import os | |
| import shutil | |
| import sys | |
| import cv2 | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| ''' | |
| 0,0 ------> x (width) | |
| | | |
| | (Left,Top) | |
| | *_________ | |
| | | | | |
| | | | |
| y |_________| | |
| (height) * | |
| (Right,Bottom) | |
| ''' | |
| def log_average_miss_rate(precision, fp_cumsum, num_images): | |
| """ | |
| log-average miss rate: | |
| Calculated by averaging miss rates at 9 evenly spaced FPPI points | |
| between 10e-2 and 10e0, in log-space. | |
| output: | |
| lamr | log-average miss rate | |
| mr | miss rate | |
| fppi | false positives per image | |
| references: | |
| [1] Dollar, Piotr, et al. "Pedestrian Detection: An Evaluation of the | |
| State of the Art." Pattern Analysis and Machine Intelligence, IEEE | |
| Transactions on 34.4 (2012): 743 - 761. | |
| """ | |
| if precision.size == 0: | |
| lamr = 0 | |
| mr = 1 | |
| fppi = 0 | |
| return lamr, mr, fppi | |
| fppi = fp_cumsum / float(num_images) | |
| mr = (1 - precision) | |
| fppi_tmp = np.insert(fppi, 0, -1.0) | |
| mr_tmp = np.insert(mr, 0, 1.0) | |
| ref = np.logspace(-2.0, 0.0, num = 9) | |
| for i, ref_i in enumerate(ref): | |
| j = np.where(fppi_tmp <= ref_i)[-1][-1] | |
| ref[i] = mr_tmp[j] | |
| lamr = math.exp(np.mean(np.log(np.maximum(1e-10, ref)))) | |
| return lamr, mr, fppi | |
| """ | |
| throw error and exit | |
| """ | |
| def error(msg): | |
| print(msg) | |
| sys.exit(0) | |
| """ | |
| check if the number is a float between 0.0 and 1.0 | |
| """ | |
| def is_float_between_0_and_1(value): | |
| try: | |
| val = float(value) | |
| if val > 0.0 and val < 1.0: | |
| return True | |
| else: | |
| return False | |
| except ValueError: | |
| return False | |
| """ | |
| Calculate the AP given the recall and precision array | |
| 1st) We compute a version of the measured precision/recall curve with | |
| precision monotonically decreasing | |
| 2nd) We compute the AP as the area under this curve by numerical integration. | |
| """ | |
| def voc_ap(rec, prec): | |
| """ | |
| --- Official matlab code VOC2012--- | |
| mrec=[0 ; rec ; 1]; | |
| mpre=[0 ; prec ; 0]; | |
| for i=numel(mpre)-1:-1:1 | |
| mpre(i)=max(mpre(i),mpre(i+1)); | |
| end | |
| i=find(mrec(2:end)~=mrec(1:end-1))+1; | |
| ap=sum((mrec(i)-mrec(i-1)).*mpre(i)); | |
| """ | |
| rec.insert(0, 0.0) # insert 0.0 at begining of list | |
| rec.append(1.0) # insert 1.0 at end of list | |
| mrec = rec[:] | |
| prec.insert(0, 0.0) # insert 0.0 at begining of list | |
| prec.append(0.0) # insert 0.0 at end of list | |
| mpre = prec[:] | |
| """ | |
| This part makes the precision monotonically decreasing | |
| (goes from the end to the beginning) | |
| matlab: for i=numel(mpre)-1:-1:1 | |
| mpre(i)=max(mpre(i),mpre(i+1)); | |
| """ | |
| for i in range(len(mpre)-2, -1, -1): | |
| mpre[i] = max(mpre[i], mpre[i+1]) | |
| """ | |
| This part creates a list of indexes where the recall changes | |
| matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1; | |
| """ | |
| i_list = [] | |
| for i in range(1, len(mrec)): | |
| if mrec[i] != mrec[i-1]: | |
| i_list.append(i) # if it was matlab would be i + 1 | |
| """ | |
| The Average Precision (AP) is the area under the curve | |
| (numerical integration) | |
| matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i)); | |
| """ | |
| ap = 0.0 | |
| for i in i_list: | |
| ap += ((mrec[i]-mrec[i-1])*mpre[i]) | |
| return ap, mrec, mpre | |
| """ | |
| Convert the lines of a file to a list | |
| """ | |
| def file_lines_to_list(path): | |
| # open txt file lines to a list | |
| with open(path) as f: | |
| content = f.readlines() | |
| # remove whitespace characters like `\n` at the end of each line | |
| content = [x.strip() for x in content] | |
| return content | |
| """ | |
| Draws text in image | |
| """ | |
| def draw_text_in_image(img, text, pos, color, line_width): | |
| font = cv2.FONT_HERSHEY_PLAIN | |
| fontScale = 1 | |
| lineType = 1 | |
| bottomLeftCornerOfText = pos | |
| cv2.putText(img, text, | |
| bottomLeftCornerOfText, | |
| font, | |
| fontScale, | |
| color, | |
| lineType) | |
| text_width, _ = cv2.getTextSize(text, font, fontScale, lineType)[0] | |
| return img, (line_width + text_width) | |
| """ | |
| Plot - adjust axes | |
| """ | |
| def adjust_axes(r, t, fig, axes): | |
| # get text width for re-scaling | |
| bb = t.get_window_extent(renderer=r) | |
| text_width_inches = bb.width / fig.dpi | |
| # get axis width in inches | |
| current_fig_width = fig.get_figwidth() | |
| new_fig_width = current_fig_width + text_width_inches | |
| propotion = new_fig_width / current_fig_width | |
| # get axis limit | |
| x_lim = axes.get_xlim() | |
| axes.set_xlim([x_lim[0], x_lim[1]*propotion]) | |
| """ | |
| Draw plot using Matplotlib | |
| """ | |
| def draw_plot_func(dictionary, n_classes, window_title, plot_title, x_label, output_path, to_show, plot_color, true_p_bar): | |
| # sort the dictionary by decreasing value, into a list of tuples | |
| sorted_dic_by_value = sorted(dictionary.items(), key=operator.itemgetter(1)) | |
| # unpacking the list of tuples into two lists | |
| sorted_keys, sorted_values = zip(*sorted_dic_by_value) | |
| # | |
| if true_p_bar != "": | |
| """ | |
| Special case to draw in: | |
| - green -> TP: True Positives (object detected and matches ground-truth) | |
| - red -> FP: False Positives (object detected but does not match ground-truth) | |
| - orange -> FN: False Negatives (object not detected but present in the ground-truth) | |
| """ | |
| fp_sorted = [] | |
| tp_sorted = [] | |
| for key in sorted_keys: | |
| fp_sorted.append(dictionary[key] - true_p_bar[key]) | |
| tp_sorted.append(true_p_bar[key]) | |
| plt.barh(range(n_classes), fp_sorted, align='center', color='crimson', label='False Positive') | |
| plt.barh(range(n_classes), tp_sorted, align='center', color='forestgreen', label='True Positive', left=fp_sorted) | |
| # add legend | |
| plt.legend(loc='lower right') | |
| """ | |
| Write number on side of bar | |
| """ | |
| fig = plt.gcf() # gcf - get current figure | |
| axes = plt.gca() | |
| r = fig.canvas.get_renderer() | |
| for i, val in enumerate(sorted_values): | |
| fp_val = fp_sorted[i] | |
| tp_val = tp_sorted[i] | |
| fp_str_val = " " + str(fp_val) | |
| tp_str_val = fp_str_val + " " + str(tp_val) | |
| # trick to paint multicolor with offset: | |
| # first paint everything and then repaint the first number | |
| t = plt.text(val, i, tp_str_val, color='forestgreen', va='center', fontweight='bold') | |
| plt.text(val, i, fp_str_val, color='crimson', va='center', fontweight='bold') | |
| if i == (len(sorted_values)-1): # largest bar | |
| adjust_axes(r, t, fig, axes) | |
| else: | |
| plt.barh(range(n_classes), sorted_values, color=plot_color) | |
| """ | |
| Write number on side of bar | |
| """ | |
| fig = plt.gcf() # gcf - get current figure | |
| axes = plt.gca() | |
| r = fig.canvas.get_renderer() | |
| for i, val in enumerate(sorted_values): | |
| str_val = " " + str(val) # add a space before | |
| if val < 1.0: | |
| str_val = " {0:.2f}".format(val) | |
| t = plt.text(val, i, str_val, color=plot_color, va='center', fontweight='bold') | |
| # re-set axes to show number inside the figure | |
| if i == (len(sorted_values)-1): # largest bar | |
| adjust_axes(r, t, fig, axes) | |
| # set window title | |
| fig.canvas.set_window_title(window_title) | |
| # write classes in y axis | |
| tick_font_size = 12 | |
| plt.yticks(range(n_classes), sorted_keys, fontsize=tick_font_size) | |
| """ | |
| Re-scale height accordingly | |
| """ | |
| init_height = fig.get_figheight() | |
| # comput the matrix height in points and inches | |
| dpi = fig.dpi | |
| height_pt = n_classes * (tick_font_size * 1.4) # 1.4 (some spacing) | |
| height_in = height_pt / dpi | |
| # compute the required figure height | |
| top_margin = 0.15 # in percentage of the figure height | |
| bottom_margin = 0.05 # in percentage of the figure height | |
| figure_height = height_in / (1 - top_margin - bottom_margin) | |
| # set new height | |
| if figure_height > init_height: | |
| fig.set_figheight(figure_height) | |
| # set plot title | |
| plt.title(plot_title, fontsize=14) | |
| # set axis titles | |
| # plt.xlabel('classes') | |
| plt.xlabel(x_label, fontsize='large') | |
| # adjust size of window | |
| fig.tight_layout() | |
| # save the plot | |
| fig.savefig(output_path) | |
| # show image | |
| if to_show: | |
| plt.show() | |
| # close the plot | |
| plt.close() | |
| def get_map(MINOVERLAP, draw_plot, path = './map_out'): | |
| GT_PATH = os.path.join(path, 'ground-truth') | |
| DR_PATH = os.path.join(path, 'detection-results') | |
| IMG_PATH = os.path.join(path, 'images-optional') | |
| TEMP_FILES_PATH = os.path.join(path, '.temp_files') | |
| RESULTS_FILES_PATH = os.path.join(path, 'results') | |
| show_animation = True | |
| if os.path.exists(IMG_PATH): | |
| for dirpath, dirnames, files in os.walk(IMG_PATH): | |
| if not files: | |
| show_animation = False | |
| else: | |
| show_animation = False | |
| if not os.path.exists(TEMP_FILES_PATH): | |
| os.makedirs(TEMP_FILES_PATH) | |
| if os.path.exists(RESULTS_FILES_PATH): | |
| shutil.rmtree(RESULTS_FILES_PATH) | |
| if draw_plot: | |
| os.makedirs(os.path.join(RESULTS_FILES_PATH, "AP")) | |
| os.makedirs(os.path.join(RESULTS_FILES_PATH, "F1")) | |
| os.makedirs(os.path.join(RESULTS_FILES_PATH, "Recall")) | |
| os.makedirs(os.path.join(RESULTS_FILES_PATH, "Precision")) | |
| if show_animation: | |
| os.makedirs(os.path.join(RESULTS_FILES_PATH, "images", "detections_one_by_one")) | |
| ground_truth_files_list = glob.glob(GT_PATH + '/*.txt') | |
| if len(ground_truth_files_list) == 0: | |
| error("Error: No ground-truth files found!") | |
| ground_truth_files_list.sort() | |
| gt_counter_per_class = {} | |
| counter_images_per_class = {} | |
| for txt_file in ground_truth_files_list: | |
| file_id = txt_file.split(".txt", 1)[0] | |
| file_id = os.path.basename(os.path.normpath(file_id)) | |
| temp_path = os.path.join(DR_PATH, (file_id + ".txt")) | |
| if not os.path.exists(temp_path): | |
| error_msg = "Error. File not found: {}\n".format(temp_path) | |
| error(error_msg) | |
| lines_list = file_lines_to_list(txt_file) | |
| bounding_boxes = [] | |
| is_difficult = False | |
| already_seen_classes = [] | |
| for line in lines_list: | |
| try: | |
| if "difficult" in line: | |
| class_name, left, top, right, bottom, _difficult = line.split() | |
| is_difficult = True | |
| else: | |
| class_name, left, top, right, bottom = line.split() | |
| except: | |
| if "difficult" in line: | |
| line_split = line.split() | |
| _difficult = line_split[-1] | |
| bottom = line_split[-2] | |
| right = line_split[-3] | |
| top = line_split[-4] | |
| left = line_split[-5] | |
| class_name = "" | |
| for name in line_split[:-5]: | |
| class_name += name + " " | |
| class_name = class_name[:-1] | |
| is_difficult = True | |
| else: | |
| line_split = line.split() | |
| bottom = line_split[-1] | |
| right = line_split[-2] | |
| top = line_split[-3] | |
| left = line_split[-4] | |
| class_name = "" | |
| for name in line_split[:-4]: | |
| class_name += name + " " | |
| class_name = class_name[:-1] | |
| bbox = left + " " + top + " " + right + " " + bottom | |
| if is_difficult: | |
| bounding_boxes.append({"class_name":class_name, "bbox":bbox, "used":False, "difficult":True}) | |
| is_difficult = False | |
| else: | |
| bounding_boxes.append({"class_name":class_name, "bbox":bbox, "used":False}) | |
| if class_name in gt_counter_per_class: | |
| gt_counter_per_class[class_name] += 1 | |
| else: | |
| gt_counter_per_class[class_name] = 1 | |
| if class_name not in already_seen_classes: | |
| if class_name in counter_images_per_class: | |
| counter_images_per_class[class_name] += 1 | |
| else: | |
| counter_images_per_class[class_name] = 1 | |
| already_seen_classes.append(class_name) | |
| with open(TEMP_FILES_PATH + "/" + file_id + "_ground_truth.json", 'w') as outfile: | |
| json.dump(bounding_boxes, outfile) | |
| gt_classes = list(gt_counter_per_class.keys()) | |
| gt_classes = sorted(gt_classes) | |
| n_classes = len(gt_classes) | |
| dr_files_list = glob.glob(DR_PATH + '/*.txt') | |
| dr_files_list.sort() | |
| for class_index, class_name in enumerate(gt_classes): | |
| bounding_boxes = [] | |
| for txt_file in dr_files_list: | |
| file_id = txt_file.split(".txt",1)[0] | |
| file_id = os.path.basename(os.path.normpath(file_id)) | |
| temp_path = os.path.join(GT_PATH, (file_id + ".txt")) | |
| if class_index == 0: | |
| if not os.path.exists(temp_path): | |
| error_msg = "Error. File not found: {}\n".format(temp_path) | |
| error(error_msg) | |
| lines = file_lines_to_list(txt_file) | |
| for line in lines: | |
| try: | |
| tmp_class_name, confidence, left, top, right, bottom = line.split() | |
| except: | |
| line_split = line.split() | |
| bottom = line_split[-1] | |
| right = line_split[-2] | |
| top = line_split[-3] | |
| left = line_split[-4] | |
| confidence = line_split[-5] | |
| tmp_class_name = "" | |
| for name in line_split[:-5]: | |
| tmp_class_name += name + " " | |
| tmp_class_name = tmp_class_name[:-1] | |
| if tmp_class_name == class_name: | |
| bbox = left + " " + top + " " + right + " " +bottom | |
| bounding_boxes.append({"confidence":confidence, "file_id":file_id, "bbox":bbox}) | |
| bounding_boxes.sort(key=lambda x:float(x['confidence']), reverse=True) | |
| with open(TEMP_FILES_PATH + "/" + class_name + "_dr.json", 'w') as outfile: | |
| json.dump(bounding_boxes, outfile) | |
| sum_AP = 0.0 | |
| ap_dictionary = {} | |
| lamr_dictionary = {} | |
| with open(RESULTS_FILES_PATH + "/results.txt", 'w') as results_file: | |
| results_file.write("# AP and precision/recall per class\n") | |
| count_true_positives = {} | |
| for class_index, class_name in enumerate(gt_classes): | |
| count_true_positives[class_name] = 0 | |
| dr_file = TEMP_FILES_PATH + "/" + class_name + "_dr.json" | |
| dr_data = json.load(open(dr_file)) | |
| nd = len(dr_data) | |
| tp = [0] * nd | |
| fp = [0] * nd | |
| score = [0] * nd | |
| score05_idx = 0 | |
| for idx, detection in enumerate(dr_data): | |
| file_id = detection["file_id"] | |
| score[idx] = float(detection["confidence"]) | |
| if score[idx] > 0.5: | |
| score05_idx = idx | |
| if show_animation: | |
| ground_truth_img = glob.glob1(IMG_PATH, file_id + ".*") | |
| if len(ground_truth_img) == 0: | |
| error("Error. Image not found with id: " + file_id) | |
| elif len(ground_truth_img) > 1: | |
| error("Error. Multiple image with id: " + file_id) | |
| else: | |
| img = cv2.imread(IMG_PATH + "/" + ground_truth_img[0]) | |
| img_cumulative_path = RESULTS_FILES_PATH + "/images/" + ground_truth_img[0] | |
| if os.path.isfile(img_cumulative_path): | |
| img_cumulative = cv2.imread(img_cumulative_path) | |
| else: | |
| img_cumulative = img.copy() | |
| bottom_border = 60 | |
| BLACK = [0, 0, 0] | |
| img = cv2.copyMakeBorder(img, 0, bottom_border, 0, 0, cv2.BORDER_CONSTANT, value=BLACK) | |
| gt_file = TEMP_FILES_PATH + "/" + file_id + "_ground_truth.json" | |
| ground_truth_data = json.load(open(gt_file)) | |
| ovmax = -1 | |
| gt_match = -1 | |
| bb = [float(x) for x in detection["bbox"].split()] | |
| for obj in ground_truth_data: | |
| if obj["class_name"] == class_name: | |
| bbgt = [ float(x) for x in obj["bbox"].split() ] | |
| bi = [max(bb[0],bbgt[0]), max(bb[1],bbgt[1]), min(bb[2],bbgt[2]), min(bb[3],bbgt[3])] | |
| iw = bi[2] - bi[0] + 1 | |
| ih = bi[3] - bi[1] + 1 | |
| if iw > 0 and ih > 0: | |
| ua = (bb[2] - bb[0] + 1) * (bb[3] - bb[1] + 1) + (bbgt[2] - bbgt[0] | |
| + 1) * (bbgt[3] - bbgt[1] + 1) - iw * ih | |
| ov = iw * ih / ua | |
| if ov > ovmax: | |
| ovmax = ov | |
| gt_match = obj | |
| if show_animation: | |
| status = "NO MATCH FOUND!" | |
| min_overlap = MINOVERLAP | |
| if ovmax >= min_overlap: | |
| if "difficult" not in gt_match: | |
| if not bool(gt_match["used"]): | |
| tp[idx] = 1 | |
| gt_match["used"] = True | |
| count_true_positives[class_name] += 1 | |
| with open(gt_file, 'w') as f: | |
| f.write(json.dumps(ground_truth_data)) | |
| if show_animation: | |
| status = "MATCH!" | |
| else: | |
| fp[idx] = 1 | |
| if show_animation: | |
| status = "REPEATED MATCH!" | |
| else: | |
| fp[idx] = 1 | |
| if ovmax > 0: | |
| status = "INSUFFICIENT OVERLAP" | |
| """ | |
| Draw image to show animation | |
| """ | |
| if show_animation: | |
| height, widht = img.shape[:2] | |
| white = (255,255,255) | |
| light_blue = (255,200,100) | |
| green = (0,255,0) | |
| light_red = (30,30,255) | |
| margin = 10 | |
| # 1nd line | |
| v_pos = int(height - margin - (bottom_border / 2.0)) | |
| text = "Image: " + ground_truth_img[0] + " " | |
| img, line_width = draw_text_in_image(img, text, (margin, v_pos), white, 0) | |
| text = "Class [" + str(class_index) + "/" + str(n_classes) + "]: " + class_name + " " | |
| img, line_width = draw_text_in_image(img, text, (margin + line_width, v_pos), light_blue, line_width) | |
| if ovmax != -1: | |
| color = light_red | |
| if status == "INSUFFICIENT OVERLAP": | |
| text = "IoU: {0:.2f}% ".format(ovmax*100) + "< {0:.2f}% ".format(min_overlap*100) | |
| else: | |
| text = "IoU: {0:.2f}% ".format(ovmax*100) + ">= {0:.2f}% ".format(min_overlap*100) | |
| color = green | |
| img, _ = draw_text_in_image(img, text, (margin + line_width, v_pos), color, line_width) | |
| # 2nd line | |
| v_pos += int(bottom_border / 2.0) | |
| rank_pos = str(idx+1) | |
| text = "Detection #rank: " + rank_pos + " confidence: {0:.2f}% ".format(float(detection["confidence"])*100) | |
| img, line_width = draw_text_in_image(img, text, (margin, v_pos), white, 0) | |
| color = light_red | |
| if status == "MATCH!": | |
| color = green | |
| text = "Result: " + status + " " | |
| img, line_width = draw_text_in_image(img, text, (margin + line_width, v_pos), color, line_width) | |
| font = cv2.FONT_HERSHEY_SIMPLEX | |
| if ovmax > 0: | |
| bbgt = [ int(round(float(x))) for x in gt_match["bbox"].split() ] | |
| cv2.rectangle(img,(bbgt[0],bbgt[1]),(bbgt[2],bbgt[3]),light_blue,2) | |
| cv2.rectangle(img_cumulative,(bbgt[0],bbgt[1]),(bbgt[2],bbgt[3]),light_blue,2) | |
| cv2.putText(img_cumulative, class_name, (bbgt[0],bbgt[1] - 5), font, 0.6, light_blue, 1, cv2.LINE_AA) | |
| bb = [int(i) for i in bb] | |
| cv2.rectangle(img,(bb[0],bb[1]),(bb[2],bb[3]),color,2) | |
| cv2.rectangle(img_cumulative,(bb[0],bb[1]),(bb[2],bb[3]),color,2) | |
| cv2.putText(img_cumulative, class_name, (bb[0],bb[1] - 5), font, 0.6, color, 1, cv2.LINE_AA) | |
| cv2.imshow("Animation", img) | |
| cv2.waitKey(20) | |
| output_img_path = RESULTS_FILES_PATH + "/images/detections_one_by_one/" + class_name + "_detection" + str(idx) + ".jpg" | |
| cv2.imwrite(output_img_path, img) | |
| cv2.imwrite(img_cumulative_path, img_cumulative) | |
| cumsum = 0 | |
| for idx, val in enumerate(fp): | |
| fp[idx] += cumsum | |
| cumsum += val | |
| cumsum = 0 | |
| for idx, val in enumerate(tp): | |
| tp[idx] += cumsum | |
| cumsum += val | |
| rec = tp[:] | |
| for idx, val in enumerate(tp): | |
| rec[idx] = float(tp[idx]) / np.maximum(gt_counter_per_class[class_name], 1) | |
| prec = tp[:] | |
| for idx, val in enumerate(tp): | |
| prec[idx] = float(tp[idx]) / np.maximum((fp[idx] + tp[idx]), 1) | |
| ap, mrec, mprec = voc_ap(rec[:], prec[:]) | |
| F1 = np.array(rec)*np.array(prec)*2 / np.where((np.array(prec)+np.array(rec))==0, 1, (np.array(prec)+np.array(rec))) | |
| sum_AP += ap | |
| text = "{0:.2f}%".format(ap*100) + " = " + class_name + " AP " #class_name + " AP = {0:.2f}%".format(ap*100) | |
| if len(prec)>0: | |
| F1_text = "{0:.2f}".format(F1[score05_idx]) + " = " + class_name + " F1 " | |
| Recall_text = "{0:.2f}%".format(rec[score05_idx]*100) + " = " + class_name + " Recall " | |
| Precision_text = "{0:.2f}%".format(prec[score05_idx]*100) + " = " + class_name + " Precision " | |
| else: | |
| F1_text = "0.00" + " = " + class_name + " F1 " | |
| Recall_text = "0.00%" + " = " + class_name + " Recall " | |
| Precision_text = "0.00%" + " = " + class_name + " Precision " | |
| rounded_prec = [ '%.2f' % elem for elem in prec ] | |
| rounded_rec = [ '%.2f' % elem for elem in rec ] | |
| results_file.write(text + "\n Precision: " + str(rounded_prec) + "\n Recall :" + str(rounded_rec) + "\n\n") | |
| if len(prec)>0: | |
| print(text + "\t||\tscore_threhold=0.5 : " + "F1=" + "{0:.2f}".format(F1[score05_idx])\ | |
| + " ; Recall=" + "{0:.2f}%".format(rec[score05_idx]*100) + " ; Precision=" + "{0:.2f}%".format(prec[score05_idx]*100)) | |
| else: | |
| print(text + "\t||\tscore_threhold=0.5 : F1=0.00% ; Recall=0.00% ; Precision=0.00%") | |
| ap_dictionary[class_name] = ap | |
| n_images = counter_images_per_class[class_name] | |
| lamr, mr, fppi = log_average_miss_rate(np.array(rec), np.array(fp), n_images) | |
| lamr_dictionary[class_name] = lamr | |
| if draw_plot: | |
| plt.plot(rec, prec, '-o') | |
| area_under_curve_x = mrec[:-1] + [mrec[-2]] + [mrec[-1]] | |
| area_under_curve_y = mprec[:-1] + [0.0] + [mprec[-1]] | |
| plt.fill_between(area_under_curve_x, 0, area_under_curve_y, alpha=0.2, edgecolor='r') | |
| fig = plt.gcf() | |
| fig.canvas.set_window_title('AP ' + class_name) | |
| plt.title('class: ' + text) | |
| plt.xlabel('Recall') | |
| plt.ylabel('Precision') | |
| axes = plt.gca() | |
| axes.set_xlim([0.0,1.0]) | |
| axes.set_ylim([0.0,1.05]) | |
| fig.savefig(RESULTS_FILES_PATH + "/AP/" + class_name + ".png") | |
| plt.cla() | |
| plt.plot(score, F1, "-", color='orangered') | |
| plt.title('class: ' + F1_text + "\nscore_threhold=0.5") | |
| plt.xlabel('Score_Threhold') | |
| plt.ylabel('F1') | |
| axes = plt.gca() | |
| axes.set_xlim([0.0,1.0]) | |
| axes.set_ylim([0.0,1.05]) | |
| fig.savefig(RESULTS_FILES_PATH + "/F1/" + class_name + ".png") | |
| plt.cla() | |
| plt.plot(score, rec, "-H", color='gold') | |
| plt.title('class: ' + Recall_text + "\nscore_threhold=0.5") | |
| plt.xlabel('Score_Threhold') | |
| plt.ylabel('Recall') | |
| axes = plt.gca() | |
| axes.set_xlim([0.0,1.0]) | |
| axes.set_ylim([0.0,1.05]) | |
| fig.savefig(RESULTS_FILES_PATH + "/Recall/" + class_name + ".png") | |
| plt.cla() | |
| plt.plot(score, prec, "-s", color='palevioletred') | |
| plt.title('class: ' + Precision_text + "\nscore_threhold=0.5") | |
| plt.xlabel('Score_Threhold') | |
| plt.ylabel('Precision') | |
| axes = plt.gca() | |
| axes.set_xlim([0.0,1.0]) | |
| axes.set_ylim([0.0,1.05]) | |
| fig.savefig(RESULTS_FILES_PATH + "/Precision/" + class_name + ".png") | |
| plt.cla() | |
| if show_animation: | |
| cv2.destroyAllWindows() | |
| results_file.write("\n# mAP of all classes\n") | |
| mAP = sum_AP / n_classes | |
| text = "mAP = {0:.2f}%".format(mAP*100) | |
| results_file.write(text + "\n") | |
| print(text) | |
| shutil.rmtree(TEMP_FILES_PATH) | |
| """ | |
| Count total of detection-results | |
| """ | |
| det_counter_per_class = {} | |
| for txt_file in dr_files_list: | |
| lines_list = file_lines_to_list(txt_file) | |
| for line in lines_list: | |
| class_name = line.split()[0] | |
| if class_name in det_counter_per_class: | |
| det_counter_per_class[class_name] += 1 | |
| else: | |
| det_counter_per_class[class_name] = 1 | |
| dr_classes = list(det_counter_per_class.keys()) | |
| """ | |
| Write number of ground-truth objects per class to results.txt | |
| """ | |
| with open(RESULTS_FILES_PATH + "/results.txt", 'a') as results_file: | |
| results_file.write("\n# Number of ground-truth objects per class\n") | |
| for class_name in sorted(gt_counter_per_class): | |
| results_file.write(class_name + ": " + str(gt_counter_per_class[class_name]) + "\n") | |
| """ | |
| Finish counting true positives | |
| """ | |
| for class_name in dr_classes: | |
| if class_name not in gt_classes: | |
| count_true_positives[class_name] = 0 | |
| """ | |
| Write number of detected objects per class to results.txt | |
| """ | |
| with open(RESULTS_FILES_PATH + "/results.txt", 'a') as results_file: | |
| results_file.write("\n# Number of detected objects per class\n") | |
| for class_name in sorted(dr_classes): | |
| n_det = det_counter_per_class[class_name] | |
| text = class_name + ": " + str(n_det) | |
| text += " (tp:" + str(count_true_positives[class_name]) + "" | |
| text += ", fp:" + str(n_det - count_true_positives[class_name]) + ")\n" | |
| results_file.write(text) | |
| """ | |
| Plot the total number of occurences of each class in the ground-truth | |
| """ | |
| if draw_plot: | |
| window_title = "ground-truth-info" | |
| plot_title = "ground-truth\n" | |
| plot_title += "(" + str(len(ground_truth_files_list)) + " files and " + str(n_classes) + " classes)" | |
| x_label = "Number of objects per class" | |
| output_path = RESULTS_FILES_PATH + "/ground-truth-info.png" | |
| to_show = False | |
| plot_color = 'forestgreen' | |
| draw_plot_func( | |
| gt_counter_per_class, | |
| n_classes, | |
| window_title, | |
| plot_title, | |
| x_label, | |
| output_path, | |
| to_show, | |
| plot_color, | |
| '', | |
| ) | |
| # """ | |
| # Plot the total number of occurences of each class in the "detection-results" folder | |
| # """ | |
| # if draw_plot: | |
| # window_title = "detection-results-info" | |
| # # Plot title | |
| # plot_title = "detection-results\n" | |
| # plot_title += "(" + str(len(dr_files_list)) + " files and " | |
| # count_non_zero_values_in_dictionary = sum(int(x) > 0 for x in list(det_counter_per_class.values())) | |
| # plot_title += str(count_non_zero_values_in_dictionary) + " detected classes)" | |
| # # end Plot title | |
| # x_label = "Number of objects per class" | |
| # output_path = RESULTS_FILES_PATH + "/detection-results-info.png" | |
| # to_show = False | |
| # plot_color = 'forestgreen' | |
| # true_p_bar = count_true_positives | |
| # draw_plot_func( | |
| # det_counter_per_class, | |
| # len(det_counter_per_class), | |
| # window_title, | |
| # plot_title, | |
| # x_label, | |
| # output_path, | |
| # to_show, | |
| # plot_color, | |
| # true_p_bar | |
| # ) | |
| """ | |
| Draw log-average miss rate plot (Show lamr of all classes in decreasing order) | |
| """ | |
| if draw_plot: | |
| window_title = "lamr" | |
| plot_title = "log-average miss rate" | |
| x_label = "log-average miss rate" | |
| output_path = RESULTS_FILES_PATH + "/lamr.png" | |
| to_show = False | |
| plot_color = 'royalblue' | |
| draw_plot_func( | |
| lamr_dictionary, | |
| n_classes, | |
| window_title, | |
| plot_title, | |
| x_label, | |
| output_path, | |
| to_show, | |
| plot_color, | |
| "" | |
| ) | |
| """ | |
| Draw mAP plot (Show AP's of all classes in decreasing order) | |
| """ | |
| if draw_plot: | |
| window_title = "mAP" | |
| plot_title = "mAP = {0:.2f}%".format(mAP*100) | |
| x_label = "Average Precision" | |
| output_path = RESULTS_FILES_PATH + "/mAP.png" | |
| to_show = True | |
| plot_color = 'royalblue' | |
| draw_plot_func( | |
| ap_dictionary, | |
| n_classes, | |
| window_title, | |
| plot_title, | |
| x_label, | |
| output_path, | |
| to_show, | |
| plot_color, | |
| "" | |
| ) | |
| def preprocess_gt(gt_path, class_names): | |
| image_ids = os.listdir(gt_path) | |
| results = {} | |
| images = [] | |
| bboxes = [] | |
| for i, image_id in enumerate(image_ids): | |
| lines_list = file_lines_to_list(os.path.join(gt_path, image_id)) | |
| boxes_per_image = [] | |
| image = {} | |
| image_id = os.path.splitext(image_id)[0] | |
| image['file_name'] = image_id + '.jpg' | |
| image['width'] = 1 | |
| image['height'] = 1 | |
| #-----------------------------------------------------------------# | |
| # 感谢 多学学英语吧 的提醒 | |
| # 解决了'Results do not correspond to current coco set'问题 | |
| #-----------------------------------------------------------------# | |
| image['id'] = str(image_id) | |
| for line in lines_list: | |
| difficult = 0 | |
| if "difficult" in line: | |
| line_split = line.split() | |
| left, top, right, bottom, _difficult = line_split[-5:] | |
| class_name = "" | |
| for name in line_split[:-5]: | |
| class_name += name + " " | |
| class_name = class_name[:-1] | |
| difficult = 1 | |
| else: | |
| line_split = line.split() | |
| left, top, right, bottom = line_split[-4:] | |
| class_name = "" | |
| for name in line_split[:-4]: | |
| class_name += name + " " | |
| class_name = class_name[:-1] | |
| left, top, right, bottom = float(left), float(top), float(right), float(bottom) | |
| cls_id = class_names.index(class_name) + 1 | |
| bbox = [left, top, right - left, bottom - top, difficult, str(image_id), cls_id, (right - left) * (bottom - top) - 10.0] | |
| boxes_per_image.append(bbox) | |
| images.append(image) | |
| bboxes.extend(boxes_per_image) | |
| results['images'] = images | |
| categories = [] | |
| for i, cls in enumerate(class_names): | |
| category = {} | |
| category['supercategory'] = cls | |
| category['name'] = cls | |
| category['id'] = i + 1 | |
| categories.append(category) | |
| results['categories'] = categories | |
| annotations = [] | |
| for i, box in enumerate(bboxes): | |
| annotation = {} | |
| annotation['area'] = box[-1] | |
| annotation['category_id'] = box[-2] | |
| annotation['image_id'] = box[-3] | |
| annotation['iscrowd'] = box[-4] | |
| annotation['bbox'] = box[:4] | |
| annotation['id'] = i | |
| annotations.append(annotation) | |
| results['annotations'] = annotations | |
| return results | |
| def preprocess_dr(dr_path, class_names): | |
| image_ids = os.listdir(dr_path) | |
| results = [] | |
| for image_id in image_ids: | |
| lines_list = file_lines_to_list(os.path.join(dr_path, image_id)) | |
| image_id = os.path.splitext(image_id)[0] | |
| for line in lines_list: | |
| line_split = line.split() | |
| confidence, left, top, right, bottom = line_split[-5:] | |
| class_name = "" | |
| for name in line_split[:-5]: | |
| class_name += name + " " | |
| class_name = class_name[:-1] | |
| left, top, right, bottom = float(left), float(top), float(right), float(bottom) | |
| result = {} | |
| result["image_id"] = str(image_id) | |
| result["category_id"] = class_names.index(class_name) + 1 | |
| result["bbox"] = [left, top, right - left, bottom - top] | |
| result["score"] = float(confidence) | |
| results.append(result) | |
| return results | |
| def get_coco_map(class_names, path): | |
| from pycocotools.coco import COCO | |
| from pycocotools.cocoeval import COCOeval | |
| GT_PATH = os.path.join(path, 'ground-truth') | |
| DR_PATH = os.path.join(path, 'detection-results') | |
| COCO_PATH = os.path.join(path, 'coco_eval') | |
| if not os.path.exists(COCO_PATH): | |
| os.makedirs(COCO_PATH) | |
| GT_JSON_PATH = os.path.join(COCO_PATH, 'instances_gt.json') | |
| DR_JSON_PATH = os.path.join(COCO_PATH, 'instances_dr.json') | |
| with open(GT_JSON_PATH, "w") as f: | |
| results_gt = preprocess_gt(GT_PATH, class_names) | |
| json.dump(results_gt, f, indent=4) | |
| with open(DR_JSON_PATH, "w") as f: | |
| results_dr = preprocess_dr(DR_PATH, class_names) | |
| json.dump(results_dr, f, indent=4) | |
| cocoGt = COCO(GT_JSON_PATH) | |
| cocoDt = cocoGt.loadRes(DR_JSON_PATH) | |
| cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') | |
| cocoEval.evaluate() | |
| cocoEval.accumulate() | |
| cocoEval.summarize() | |