import project_path import torch from tqdm import tqdm from functools import partial import numpy as np import json import time from unittest.mock import patch # assumes yolov5 on sys.path from lib.yolov5.models.experimental import attempt_load from lib.yolov5.utils.torch_utils import select_device from lib.yolov5.utils.general import clip_boxes, scale_boxes, xywh2xyxy from lib.yolov5.utils.metrics import box_iou import torch import torchvision from lib.fish_eye.tracker import Tracker ### Configuration options WEIGHTS = 'models/v5m_896_300best.pt' # will need to configure these based on GPU hardware BATCH_SIZE = 32 CONF_THRES = 0.3 # detection NMS_IOU = 0.3 # NMS IOU MIN_LENGTH = 0.3 # minimum fish length, in meters MAX_AGE = 20 # time until missing fish get's new id IOU_THRES = 0.01 # IOU threshold for tracking MIN_HITS = 11 # minimum number of frames with a specific fish for it to count ### def norm(bbox, w, h): """ Normalize a bounding box. Args: bbox: list of length 4. Can be [x,y,w,h] or [x0,y0,x1,y1] w: image width h: image height """ bb = bbox.copy() bb[0] /= w bb[1] /= h bb[2] /= w bb[3] /= h return bb def do_full_inference(dataloader, image_meter_width, image_meter_height, gp=None, weights=WEIGHTS): model, device = setup_model(weights) load = False save = False if load: with open('static/example/inference_output.json', 'r') as f: json_object = json.load(f) inference = json_object['inference'] width = json_object['width'] height = json_object['height'] else: inference, width, height = do_detection(dataloader, model, device, gp=gp) if save: json_object = { 'inference': inference, 'width': width, 'height': height } json_text = json.dumps(json_object, indent=4) with open('static/example/inference_output.json', 'w') as f: f.write(json_text) return all_preds, real_width, real_height = do_suppression(dataloader, inference, width, height, gp=gp) results = do_tracking(all_preds, image_meter_width, image_meter_height, gp=gp) return results def setup_model(weights_fp=WEIGHTS, imgsz=896, batch_size=32): if torch.cuda.is_available(): device = select_device('0', batch_size=batch_size) else: print("CUDA not available. Using CPU inference.") device = select_device('cpu', batch_size=batch_size) # Setup model for inference model = attempt_load(weights_fp, device=device) half = device.type != 'cpu' # half precision only supported on CUDA if half: model.half() model.eval() # Create dataloader for batched inference img = torch.zeros((1, 3, imgsz, imgsz), device=device) _ = model(img.half() if half else img) if device.type != 'cpu' else None # run once return model, device def do_detection(dataloader, model, device, gp=None, batch_size=BATCH_SIZE, verbose=True): """ Args: frames_dir: a directory containing frames to be evaluated image_meter_width: the width of each image, in meters (used for fish length calculation) gp: a callback function which takes as input 1 parameter, (int) percent complete prep_for_marking: re-index fish for manual marking output """ if (gp): gp(0, "Detection...") inference = [] # Run detection with tqdm(total=len(dataloader)*batch_size, desc="Running detection", ncols=0, disable=not verbose) as pbar: for batch_i, (img, _, shapes) in enumerate(dataloader): if gp: gp(batch_i / len(dataloader), pbar.__str__()) img = img.to(device, non_blocking=True) img = img.half() if device.type != 'cpu' else img.float() # uint8 to fp16/32 img /= 255.0 # 0 - 255 to 0.0 - 1.0 size = tuple(img.shape) nb, _, height, width = size # batch size, channels, height, width # Run model & NMS with torch.no_grad(): inf_out, _ = model(img, augment=False) inference.append(inf_out) pbar.update(1*batch_size) return inference, width, height def do_suppression(dataloader, inference, width, height, gp=None, batch_size=BATCH_SIZE, conf_thres=CONF_THRES, iou_thres=NMS_IOU, verbose=True): """ Args: frames_dir: a directory containing frames to be evaluated image_meter_width: the width of each image, in meters (used for fish length calculation) gp: a callback function which takes as input 1 parameter, (int) percent complete prep_for_marking: re-index fish for manual marking output """ if (gp): gp(0, "Suppression...") # keep predictions to feed them ordered into the Tracker # TODO: how to deal with large files? all_preds = {} with tqdm(total=len(dataloader)*batch_size, desc="Running suppression", ncols=0, disable=not verbose) as pbar: for batch_i, (img, _, shapes) in enumerate(dataloader): if gp: gp(batch_i / len(dataloader), pbar.__str__()) inf_out = inference[batch_i] with torch.no_grad(): output = non_max_suppression(inf_out, conf_thres=conf_thres, iou_thres=iou_thres) # Format results for si, pred in enumerate(output): # Clip boxes to image bounds and resize to input shape clip_boxes(pred, (height, width)) box = pred[:, :4].clone() # xyxy confs = pred[:, 4].clone().tolist() scale_boxes(img[si].shape[1:], box, shapes[si][0], shapes[si][1]) # to original shape # get boxes into tracker input format - normalized xyxy with confidence score # confidence score currently not used by tracker; set to 1.0 boxes = None if box.shape[0]: real_width = shapes[si][0][1] real_height = shapes[si][0][0] do_norm = partial(norm, w=shapes[si][0][1], h=shapes[si][0][0]) normed = list((map(do_norm, box[:, :4].tolist()))) boxes = np.stack([ [*bb, conf] for bb, conf in zip(normed, confs) ]) frame_num = (batch_i, si) all_preds[frame_num] = boxes pbar.update(1*batch_size) return all_preds, real_width, real_height def do_tracking(all_preds, image_meter_width, image_meter_height, gp=None, max_age=MAX_AGE, iou_thres=IOU_THRES, min_hits=MIN_HITS, min_length=MIN_LENGTH, verbose=True): if (gp): gp(0, "Tracking...") # Initialize tracker clip_info = { 'start_frame': 0, 'end_frame': len(all_preds), 'image_meter_width': image_meter_width, 'image_meter_height': image_meter_height } tracker = Tracker(clip_info, args={ 'max_age': max_age, 'min_hits': 0, 'iou_threshold': iou_thres}, min_hits=min_hits) # Run tracking with tqdm(total=len(all_preds), desc="Running tracking", ncols=0, disable=not verbose) as pbar: for i, key in enumerate(sorted(all_preds.keys())): if gp: gp(i / len(all_preds), pbar.__str__()) boxes = all_preds[key] if boxes is not None: tracker.update(boxes) else: tracker.update() pbar.update(1) json_data = tracker.finalize(min_length=min_length) return json_data @patch('json.encoder.c_make_encoder', None) def json_dump_round_float(some_object, out_path, num_digits=4): """Write a json file to disk with a specified level of precision. See: https://gist.github.com/Sukonnik-Illia/ed9b2bec1821cad437d1b8adb17406a3 """ # saving original method of = json.encoder._make_iterencode def inner(*args, **kwargs): args = list(args) # fifth argument is float formater which will we replace fmt_str = '{:.' + str(num_digits) + 'f}' args[4] = lambda o: fmt_str.format(o) return of(*args, **kwargs) with patch('json.encoder._make_iterencode', wraps=inner): return json.dump(some_object, open(out_path, 'w'), indent=2) def non_max_suppression( prediction, conf_thres=0.25, iou_thres=0.45, max_det=300, ): """Non-Maximum Suppression (NMS) on inference results to reject overlapping detections Returns: list of detections, on (n,6) tensor per image [xyxy, conf, cls] """ # Checks assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0' assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0' if isinstance(prediction, (list, tuple)): # YOLOv5 model in validation model, output = (inference_out, loss_out) prediction = prediction[0] # select only inference output device = prediction.device mps = 'mps' in device.type # Apple MPS if mps: # MPS not fully supported yet, convert tensors to CPU before NMS prediction = prediction.cpu() bs = prediction.shape[0] # batch size xc = prediction[..., 4] > conf_thres # candidates # Settings # min_wh = 2 # (pixels) minimum box width and height max_nms = 30000 # maximum number of boxes into torchvision.ops.nms() redundant = True # require redundant detections merge = False # use merge-NMS output = [torch.zeros((0, 6), device=prediction.device)] * bs for xi, x in enumerate(prediction): # image index, image inference # Keep boxes that pass confidence threshold x = x[xc[xi]] # confidence # If none remain process next image if not x.shape[0]: continue # Compute conf x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf # Box/Mask box = xywh2xyxy(x[:, :4]) # center_x, center_y, width, height) to (x1, y1, x2, y2) mask = x[:, 6:] # zero columns if no masks # Detections matrix nx6 (xyxy, conf, cls) conf, j = x[:, 5:6].max(1, keepdim=True) x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres] # Check shape n = x.shape[0] # number of boxes if not n: # no boxes continue x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence and remove excess boxes # Batched NMS boxes = x[:, :4] # boxes (offset by class), scores scores = x[:, 4] i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS i = i[:max_det] # limit detections if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix weights = iou * scores[None] # box weights x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes if redundant: i = i[iou.sum(1) > 1] # require redundancy output[xi] = x[i] if mps: output[xi] = output[xi].to(device) logging = False return output