Spaces:
Running
Running
| import numpy as np | |
| from sklearn.metrics import roc_curve, auc | |
| from sklearn.metrics import accuracy_score, balanced_accuracy_score | |
| def compute_error_auc(op_str, gt, pred, prob): | |
| # classification error | |
| pred_int = (pred > prob).astype(np.int) | |
| class_acc = (pred_int == gt).mean() * 100.0 | |
| # ROC - area under curve | |
| fpr, tpr, thresholds = roc_curve(gt, pred) | |
| roc_auc = auc(fpr, tpr) | |
| print(op_str + ", class acc = {:.3f}, ROC AUC = {:.3f}".format(class_acc, roc_auc)) | |
| #return class_acc, roc_auc | |
| def calc_average_precision(recall, precision): | |
| precision[np.isnan(precision)] = 0 | |
| recall[np.isnan(recall)] = 0 | |
| # pascal 12 way | |
| mprec = np.hstack((0, precision, 0)) | |
| mrec = np.hstack((0, recall, 1)) | |
| for ii in range(mprec.shape[0]-2, -1,-1): | |
| mprec[ii] = np.maximum(mprec[ii], mprec[ii+1]) | |
| inds = np.where(np.not_equal(mrec[1:], mrec[:-1]))[0]+1 | |
| ave_prec = ((mrec[inds] - mrec[inds-1])*mprec[inds]).sum() | |
| return float(ave_prec) | |
| def calc_recall_at_x(recall, precision, x=0.95): | |
| precision[np.isnan(precision)] = 0 | |
| recall[np.isnan(recall)] = 0 | |
| inds = np.where(precision[::-1]>x)[0] | |
| if len(inds) > 0: | |
| return float(recall[::-1][inds[0]]) | |
| else: | |
| return 0.0 | |
| def compute_affinity_1d(pred_box, gt_boxes, threshold): | |
| # first entry is start time | |
| score = np.abs(pred_box[0] - gt_boxes[:, 0]) | |
| valid_detection = np.min(score) <= threshold | |
| return valid_detection, np.argmin(score) | |
| def compute_pre_rec(gts, preds, eval_mode, class_of_interest, num_classes, threshold, ignore_start_end): | |
| """ | |
| Computes precision and recall. Assumes that each file has been exhaustively | |
| annotated. Will not count predicted detection with a start time that is within | |
| ignore_start_end miliseconds of the start or end of the file. | |
| eval_mode == 'detection' | |
| Returns overall detection results (not per class) | |
| eval_mode == 'per_class' | |
| Filters ground truth based on class of interest. This will ignore predictions | |
| assigned to gt with unknown class. | |
| eval_mode = 'top_class' | |
| Turns the problem into a binary one and selects the top predicted class | |
| for each predicted detection | |
| """ | |
| # get predictions and put in array | |
| pred_boxes = [] | |
| confidence = [] | |
| pred_class = [] | |
| file_ids = [] | |
| for pid, pp in enumerate(preds): | |
| # filter predicted calls that are too near the start or end of the file | |
| file_dur = gts[pid]['duration'] | |
| valid_inds = (pp['start_times'] >= ignore_start_end) & (pp['start_times'] <= (file_dur - ignore_start_end)) | |
| pred_boxes.append(np.vstack((pp['start_times'][valid_inds], pp['end_times'][valid_inds], | |
| pp['low_freqs'][valid_inds], pp['high_freqs'][valid_inds])).T) | |
| if eval_mode == 'detection': | |
| # overall detection | |
| confidence.append(pp['det_probs'][valid_inds]) | |
| elif eval_mode == 'per_class': | |
| # per class | |
| confidence.append(pp['class_probs'].T[valid_inds, class_of_interest]) | |
| elif eval_mode == 'top_class': | |
| # per class - note that sometimes 'class_probs' can be num_classes+1 in size | |
| top_class = np.argmax(pp['class_probs'].T[valid_inds, :num_classes], 1) | |
| confidence.append(pp['class_probs'].T[valid_inds, top_class]) | |
| pred_class.append(top_class) | |
| # be careful, assuming the order in the list is same as GT | |
| file_ids.append([pid]*valid_inds.sum()) | |
| confidence = np.hstack(confidence) | |
| file_ids = np.hstack(file_ids).astype(np.int) | |
| pred_boxes = np.vstack(pred_boxes) | |
| if len(pred_class) > 0: | |
| pred_class = np.hstack(pred_class) | |
| # extract relevant ground truth boxes | |
| gt_boxes = [] | |
| gt_assigned = [] | |
| gt_class = [] | |
| gt_generic_class = [] | |
| num_positives = 0 | |
| for gg in gts: | |
| # filter ground truth calls that are too near the start or end of the file | |
| file_dur = gg['duration'] | |
| valid_inds = (gg['start_times'] >= ignore_start_end) & (gg['start_times'] <= (file_dur - ignore_start_end)) | |
| # note, files with the incorrect duration will cause a problem | |
| if (gg['start_times'] > file_dur).sum() > 0: | |
| print('Error: file duration incorrect for', gg['id']) | |
| assert(False) | |
| boxes = np.vstack((gg['start_times'][valid_inds], gg['end_times'][valid_inds], | |
| gg['low_freqs'][valid_inds], gg['high_freqs'][valid_inds])).T | |
| gen_class = gg['class_ids'][valid_inds] == -1 | |
| class_ids = gg['class_ids'][valid_inds] | |
| # keep track of the number of relevant ground truth calls | |
| if eval_mode == 'detection': | |
| # all valid ones | |
| num_positives += len(gg['start_times'][valid_inds]) | |
| elif eval_mode == 'per_class': | |
| # all valid ones with class of interest | |
| num_positives += (gg['class_ids'][valid_inds] == class_of_interest).sum() | |
| elif eval_mode == 'top_class': | |
| # all valid ones with non generic class | |
| num_positives += (gg['class_ids'][valid_inds] > -1).sum() | |
| # find relevant classes (i.e. class_of_interest) and events without known class (i.e. generic class, -1) | |
| if eval_mode == 'per_class': | |
| class_inds = (class_ids == class_of_interest) | (class_ids == -1) | |
| boxes = boxes[class_inds, :] | |
| gen_class = gen_class[class_inds] | |
| class_ids = class_ids[class_inds] | |
| gt_assigned.append(np.zeros(boxes.shape[0])) | |
| gt_boxes.append(boxes) | |
| gt_generic_class.append(gen_class) | |
| gt_class.append(class_ids) | |
| # loop through detections and keep track of those that have been assigned | |
| true_pos = np.zeros(confidence.shape[0]) | |
| valid_inds = np.ones(confidence.shape[0]) == 1 # intialize to True | |
| sorted_inds = np.argsort(confidence)[::-1] # sort high to low | |
| for ii, ind in enumerate(sorted_inds): | |
| gt_id = file_ids[ind] | |
| valid_det = False | |
| if gt_boxes[gt_id].shape[0] > 0: | |
| # compute overlap | |
| valid_det, det_ind = compute_affinity_1d(pred_boxes[ind], gt_boxes[gt_id], | |
| threshold) | |
| # valid detection that has not already been assigned | |
| if valid_det and (gt_assigned[gt_id][det_ind] == 0): | |
| count_as_true_pos = True | |
| if eval_mode == 'top_class' and (gt_class[gt_id][det_ind] != pred_class[ind]): | |
| # needs to be the same class | |
| count_as_true_pos = False | |
| if count_as_true_pos: | |
| true_pos[ii] = 1 | |
| gt_assigned[gt_id][det_ind] = 1 | |
| # if event is generic class (i.e. gt_generic_class[gt_id][det_ind] is True) | |
| # and eval_mode != 'detection', then ignore it | |
| if gt_generic_class[gt_id][det_ind]: | |
| if eval_mode == 'per_class' or eval_mode == 'top_class': | |
| valid_inds[ii] = False | |
| # store threshold values - used for plotting | |
| conf_sorted = np.sort(confidence)[::-1][valid_inds] | |
| thresholds = np.linspace(0.1, 0.9, 9) | |
| thresholds_inds = np.zeros(len(thresholds), dtype=np.int) | |
| for ii, tt in enumerate(thresholds): | |
| thresholds_inds[ii] = np.argmin(conf_sorted > tt) | |
| thresholds_inds[thresholds_inds==0] = -1 | |
| # compute precision and recall | |
| true_pos = true_pos[valid_inds] | |
| false_pos_c = np.cumsum(1-true_pos) | |
| true_pos_c = np.cumsum(true_pos) | |
| recall = true_pos_c / num_positives | |
| precision = true_pos_c / np.maximum(true_pos_c + false_pos_c, np.finfo(np.float64).eps) | |
| results = {} | |
| results['recall'] = recall | |
| results['precision'] = precision | |
| results['num_gt'] = num_positives | |
| results['thresholds'] = thresholds | |
| results['thresholds_inds'] = thresholds_inds | |
| if num_positives == 0: | |
| results['avg_prec'] = np.nan | |
| results['rec_at_x'] = np.nan | |
| else: | |
| results['avg_prec'] = np.round(calc_average_precision(recall, precision), 5) | |
| results['rec_at_x'] = np.round(calc_recall_at_x(recall, precision), 5) | |
| return results | |
| def compute_file_accuracy_simple(gts, preds, num_classes): | |
| """ | |
| Evaluates the prediction accuracy at a file level. | |
| Does not include files that have more than one class (or the generic class). | |
| Simply chooses the class per file that has the highest probability overall. | |
| """ | |
| gt_valid = [] | |
| pred_valid = [] | |
| for ii in range(len(gts)): | |
| gt_class = np.unique(gts[ii]['class_ids']) | |
| if len(gt_class) == 1 and gt_class[0] != -1: | |
| gt_valid.append(gt_class[0]) | |
| pred = preds[ii]['class_probs'][:num_classes, :].T | |
| pred_valid.append(np.argmax(pred.mean(0))) | |
| acc = (np.array(gt_valid) == np.array(pred_valid)).mean() | |
| res = {} | |
| res['num_valid_files'] = len(gt_valid) | |
| res['num_total_files'] = len(gts) | |
| res['gt_valid_file'] = gt_valid | |
| res['pred_valid_file'] = pred_valid | |
| res['file_acc'] = np.round(acc, 5) | |
| return res | |
| def compute_file_accuracy(gts, preds, num_classes): | |
| """ | |
| Evaluates the prediction accuracy at a file level. | |
| Does not include files that have more than one class (or the unknown class). | |
| Tries several different detection thresholds and picks the best one. | |
| """ | |
| # compute min and max scoring range - then threshold | |
| min_val = 0 | |
| mins = [pp['class_probs'].min() for pp in preds if pp['class_probs'].shape[1] > 0] | |
| if len(mins) > 0: | |
| min_val = np.min(mins) | |
| max_val = 1.0 | |
| maxes = [pp['class_probs'].max() for pp in preds if pp['class_probs'].shape[1] > 0] | |
| if len(maxes) > 0: | |
| max_val = np.max(maxes) | |
| thresh = np.linspace(min_val, max_val, 11)[:10] | |
| # loop over the files and store the accuracy at different prediction thresholds | |
| # only include gt files that have one valid species | |
| gt_valid = [] | |
| pred_valid_all = [] | |
| for ii in range(len(gts)): | |
| gt_class = np.unique(gts[ii]['class_ids']) | |
| if len(gt_class) == 1 and gt_class[0] != -1: | |
| gt_valid.append(gt_class[0]) | |
| pred = preds[ii]['class_probs'][:num_classes, :].T | |
| p_class = np.zeros(len(thresh)) | |
| for tt in range(len(thresh)): | |
| p_class[tt] = (pred*(pred>=thresh[tt])).sum(0).argmax() | |
| pred_valid_all.append(p_class) | |
| # pick the result corresponding to the overall best threshold | |
| pred_valid_all = np.vstack(pred_valid_all) | |
| acc_per_thresh = (np.array(gt_valid)[..., np.newaxis] == pred_valid_all).mean(0) | |
| best_thresh = np.argmax(acc_per_thresh) | |
| best_acc = acc_per_thresh[best_thresh] | |
| pred_valid = pred_valid_all[:, best_thresh].astype(np.int).tolist() | |
| res = {} | |
| res['num_valid_files'] = len(gt_valid) | |
| res['num_total_files'] = len(gts) | |
| res['gt_valid_file'] = gt_valid | |
| res['pred_valid_file'] = pred_valid | |
| res['file_acc'] = np.round(best_acc, 5) | |
| return res | |
| def evaluate_predictions(gts, preds, class_names, detection_overlap, ignore_start_end=0.0): | |
| """ | |
| Computes metrics derived from the precision and recall. | |
| Assumes that gts and preds are both lists of the same lengths, with ground | |
| truth and predictions contained within. | |
| Returns the overall detection results, and per class results | |
| """ | |
| assert(len(gts) == len(preds)) | |
| num_classes = len(class_names) | |
| # evaluate detection on its own i.e. ignoring class | |
| det_results = compute_pre_rec(gts, preds, 'detection', None, num_classes, detection_overlap, ignore_start_end) | |
| top_class = compute_pre_rec(gts, preds, 'top_class', None, num_classes, detection_overlap, ignore_start_end) | |
| det_results['top_class'] = top_class | |
| # per class evaluation | |
| det_results['class_pr'] = [] | |
| for cc in range(num_classes): | |
| res = compute_pre_rec(gts, preds, 'per_class', cc, num_classes, detection_overlap, ignore_start_end) | |
| res['name'] = class_names[cc] | |
| det_results['class_pr'].append(res) | |
| # ignores classes that are not present in the test set | |
| det_results['avg_prec_class'] = np.mean([rs['avg_prec'] for rs in det_results['class_pr'] if rs['num_gt'] > 0]) | |
| det_results['avg_prec_class'] = np.round(det_results['avg_prec_class'], 5) | |
| # file level evaluation | |
| res_file = compute_file_accuracy(gts, preds, num_classes) | |
| det_results.update(res_file) | |
| return det_results | |