import cv2 import os import numpy as np import gradio as gr from PIL import Image from pytorch_grad_cam import GradCAM from pytorch_grad_cam.utils.image import show_cam_on_image import torch torch.set_flush_denormal(True) import torch.optim as optim from torchvision import transforms import random from collections import OrderedDict import cv2 import albumentations as A from albumentations.pytorch import ToTensorV2 import config from model import YOLOv3 from tqdm import tqdm from utils import ( mean_average_precision, cells_to_bboxes, get_evaluation_bboxes, save_checkpoint, load_checkpoint, check_class_accuracy, get_loaders, plot_couple_examples, plot_results ) from loss import YoloLoss from pytorch_lightning.callbacks import ModelSummary from pytorch_lightning import LightningModule, Trainer import warnings warnings.filterwarnings("ignore") import random torch.backends.cudnn.benchmark = True # from model import CustomResNet # from utils.utils import wrong_predictions # from utils.dataloader import get_dataloader # test_o = get_dataloader() # test_o=next(iter(test_o)) examples_dir = os.path.join(os.getcwd(), 'examples') examples = [[os.path.join(examples_dir, img), 0.5,2] for img in os.listdir(examples_dir)] num_classes = 20 IMAGE_SIZE = 416 model = YOLOv3(num_classes=num_classes) model.load_state_dict(torch.load('fourty.ckpt', map_location=torch.device('cpu'))['state_dict'])#, strict = False) # model = model.cpu() model.eval() classes = config.PASCAL_CLASSES class outdet: def __init__(self): pass def __call__(self, output): ori_in=np.unravel_index(torch.argmax(output[..., :1]), output[..., :1].shape) max_out = output[ori_in[:-1]][5:].max() # max_out = output[...,5:].max() return max_out # return output[0:1,0:1,0:1,0:1,self.target] def inference(input_img, transparency, layer_num, layerinfo): print('input recieved') import time t1 = time.time() # img_t = np.array(Image.open('examples/000008.jpg').convert("RGB").resize((416,416))) img_t=cv2.resize(input_img,(416,416)) test_transforms = A.Compose( [ A.Normalize(mean=[0, 0, 0], std=[1, 1, 1], max_pixel_value=255,), ToTensorV2(), ]) img_t=test_transforms(image=img_t)['image'] img_t=img_t.unsqueeze(0) t2 = time.time() print('preprocessing - ', str((t2-t1)), 's') # out1 = model(img_t) t3 = time.time() print('inference - ', str((t3-t2)), 's') S = [13, 26, 52] scaled_anchors = torch.tensor(config.ANCHORS) / ( 1 / torch.tensor(S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2) ) imgaa=plot_results(model,img_t, thresh=0.4, iou_thresh=0.4,anchors= scaled_anchors) print(imgaa.shape) # layer_num=10 if layerinfo == 'all': list_l = [1,2,3] else: list_l = [layer_num] t4 = time.time() print('ploting results - ', str((t4-t3)), 's') cams_im=[] for layer_num in list_l: t5 = time.time() if layer_num == 1: target_layers = [model.layers[12].conv] elif layer_num == 2: target_layers = [model.layers[11].conv] else: target_layers = [model.layers[9].conv] df = [outdet()] # cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False) # grayscale_cam = cam(input_tensor=img_t, targets=None) # grayscale_cam = grayscale_cam[0, :] cam = GradCAM(model, target_layers, use_cuda=False) grayscale_cam = cam(img_t, targets=df)[0, :] # img= np.clip(np.array(Image.open('examples/000009.jpg').convert("RGB").resize((416,416)))/255,0,1) img = np.clip(cv2.resize(cv2.cvtColor(input_img, cv2.COLOR_BGR2RGB),(416,416))/255,0,1) cam_image = show_cam_on_image(img, grayscale_cam, use_rgb=True,image_weight=transparency) cams_im.append(cam_image) t6 = time.time() print('gradcam - ', str((t6-t5)), 's') return imgaa, cams_im # return filtered_confidences, superimposed_img def get_gradcam_images(transparency): cams_ims=[] esc=list(range(10)) for i in range(3): e=random.choice(esc) esc.remove(e) input_img = cv2.imread(examples[e][0]) input_img = cv2.cvtColor(input_img, cv2.COLOR_BGR2RGB) img_t=cv2.resize(input_img,(416,416)) test_transforms = A.Compose( [ A.Normalize(mean=[0, 0, 0], std=[1, 1, 1], max_pixel_value=255,), ToTensorV2(), ]) img_t=test_transforms(image=img_t)['image'] img_t=img_t.unsqueeze(0) list_l = [1,2,3] cams_im=[] for layer_num in list_l: t5 = time.time() if layer_num == 1: target_layers = [model.layers[12].conv] elif layer_num == 2: target_layers = [model.layers[11].conv] else: target_layers = [model.layers[9].conv] df = [outdet()] # cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False) # grayscale_cam = cam(input_tensor=img_t, targets=None) # grayscale_cam = grayscale_cam[0, :] cam = GradCAM(model, target_layers, use_cuda=False) grayscale_cam = cam(img_t, targets=df)[0, :] # img= np.clip(np.array(Image.open('examples/000009.jpg').convert("RGB").resize((416,416)))/255,0,1) img = np.clip(cv2.resize(input_img,(416,416))/255,0,1) cam_image = show_cam_on_image(img, grayscale_cam, use_rgb=True,image_weight=transparency) cams_im.append(cam_image) t6 = time.time() print('gradcam - ', str((t6-t5)), 's') cams_ims.extend(cams_im) return cams_ims # demo = gr.Interface(inference, [gr.Image(shape=(32, 32)), gr.Slider(0, 1)], ["text", gr.Image(shape=(32, 32)).style(width=128, height=128)]) inference_new_image = gr.Interface( inference, inputs = [gr.Image(shape=(416, 416), label="Input Image"), gr.Slider(0, 1, value = 0.3, label="transparency?"), gr.Slider(1, 3, value = 1,step=1, label="layer?"),gr.Radio(["particular", "all"], label="layers display")], outputs = [gr.Image(shape=(416, 416), label="Model Prediction").style(width=300, height=300), gr.Gallery(label="Model gradcam")], title = 'gradio app', description = 'for dl purposes', examples = examples, ) gradcam_images = gr.Interface( get_gradcam_images, inputs = [gr.Slider(0, 1, value = 0.6, label="transparency?")], outputs = [gr.Gallery(label="gradcam sample images")], title = 'gradio app', description = 'for dl purposes' ) #gradcam_images, "grad cam images" demo = gr.TabbedInterface([inference_new_image, gradcam_images], tab_names=["Input image", "grad cam images"], title="customresnet gradcam") demo.launch( debug=True)