Repository: pytorch
Issue: Faster Inference time for first .pt model prediction
Labels: oncall: jit

## 🐛 Bug

I have created a Pytorch SSD object detection model for edge devices weighing 12.o MB. But it is taking a huge time (105 seconds of time) for initializing first-time inference. How we can deal with first-time inference lag?

Please propose some solution to making this initialized inference time shorter to not more than 0.5-2 seconds.


Sample code for doing inferences :
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 29 23:59:35 2020

@author: jraikwar
"""

import ntpath
import os
import torch
import cv2 
import numpy as np
import time
import matplotlib.pyplot as plt
from PIL import Image

import pandas as pd
import glob
import os
import shutil
from sklearn.metrics import accuracy_score, confusion_matrix


import socket    
hostname = socket.gethostname()

##### Inference Image Preprocessing #######

image_size = 300
image_mean = np.array([127, 127, 127])  # RGB layout
image_std = 128.0

def resize(image, boxes, labels):
    """
        Resize image to a fixed size
    """
    image = cv2.resize(image, (image_size,image_size))
    return image, boxes, labels

def drawBoundingBox(imgcv,result):
        for box in result:
            # print(box)
            #x1,y1,x2,y2 = (box['topleft']['x'],box['topleft']['y'],box['bottomright']['x'],box['bottomright']['y'])
            x1,y1,x2,y2 =  box[2:6]
            conf = box[1]*100
            # print(conf)
            label = str(box[0])
            # if conf < predictThresh:
            #     continue
            
            cv2.rectangle(imgcv,(x1,y1),(x2,y2),(0,255,0),6)
            labelSize=cv2.getTextSize(label,cv2.FONT_HERSHEY_COMPLEX,1,5)
            # print('labelSize>>',labelSize)
            _x1 = x1
            _y1 = y1#+int(labelSize[0][1]/2)
            _x2 = _x1+labelSize[0][0]
            _y2 = y1-int(labelSize[0][1])
            cv2.rectangle(imgcv,(_x1,_y1),(_x2,_y2),(0,255,0),cv2.FILLED)
            cv2.putText(imgcv,label,(x1,y1),cv2.FONT_HERSHEY_COMPLEX,1,(0,0,0),2)
            cv2.putText(imgcv,str('%.2f' % (conf)),(x2,y2),cv2.FONT_HERSHEY_COMPLEX,1,(255,0,0),2)
        return imgcv


def subtract_means(image, boxes, labels):
    """
        subtract mean value from the image to normalize it
    """
    mean = np.array(image_mean, dtype=np.float32)

    image = image.astype(np.float32)
    image -= mean
    return image.astype(np.float32), boxes, labels


def misc(image, boxes, labels):
    """
        Normalize image
    """
    image = image/image_std
    return image,None,None

def to_tensor(image, boxes, labels):
    """
        convert Image(numpy) to Tensor 
    """
    return torch.from_numpy(image.astype(np.float32)).permute(2, 0, 1), boxes, labels

def get_infer_transformed(image, boxes, labels,infer_transforms):

    for t in infer_transforms:
        
        image, boxes, labels = t(image, boxes, labels)

    return image

#############################################

def read_img(img_path):
    """
        read image using cv2 and convert from BGR to RGB
    """
    img = cv2.imread(img_path)
    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    return img


def get_img_label(img_name, df):
    """
    location_code,bbox values are  returned for each image name, hence multiple detection   
    """
    temp = df.loc[df['image_name']==img_name,]
    img = read_img(temp['image_path'].tolist()[0])
    mAP_calc_list = []
    for ix in temp.index.tolist():
        temp1 = temp.loc[temp.index == ix ,]
        xmin = temp1['xmin'].tolist()[0]
        ymin = temp1['ymin'].tolist()[0]
        xmax = temp1['xmax'].tolist()[0]
        ymax = temp1['ymax'].tolist()[0]
        actual_label = temp1['location_code'].tolist()[0]

        mAP_calc_list.append([actual_label,int(xmin),int(ymin),int(xmax),int(ymax)])
    return img, temp['image_path'].tolist()[0],mAP_calc_list


if __name__ == "__main__":
    
    #image_path = "C:/Users/jraikwar/OneDrive - DMI/GM/Image Annotation/users/BhagyJoe/trainWithBias/imagesWithoutRed/"
    image_path = r"C:\Users\jraik\Downloads\ai360_12th\AI_SUB_LOCATION_FRAMES/"
    #image_path = r"C:\Users\jraik\Desktop\cropped_images/"
    #image_path = r"D:\DMI\GM\training\StillImg_ValidationModel\pytorch-ssd\new_fender/"
    model_path = r"D:\DMI\Scripts/vm_1614329882_upd_doors.pt"
    #model_path = r"D:\DMI\Scripts/vm_1614105241.pt"
    #model_path = r"D:\DMI\Scripts/vm_1612464126_ffrf_best.pt"
    #model_path = r"D:\DMI\Scripts/vm_1614192879_upd_doors.pt"
    voc_txt_path = r"D:\DMI\Scripts/voc-model-labels_woffrf_upd_doors_rev_mix.txt"
    #voc_txt_path = r"D:\DMI\Scripts/voc-model-label-ffrf.txt"
    #image_new = r"D:\DMI\GM\training\StillImg_ValidationModel\pytorch-ssd/fender_pred/"
    image_new = r"C:\Users\jraik\OneDrive\Desktop\sub_loc_only"
    image_list = glob.glob(image_path+"*.*g")

    t0 = time.time()
    var_exists = 'model' in locals() or 'model' in globals()    
    if  not(var_exists):
        model = torch.jit.load(model_path)
    #model = torch.jit.load(model_path)
    print('load time {}'.format(time.time()-t0))
    list_image = []
    s=0
    v=0
    for img_name in image_list:
        #print(img_name)
        #img_path = "D:/Training/First cut model/model/Furst Cut Sub location code Model/FRONT_1G1BF5SM4J7193887_CHEVROLET_CRUZE_s8262511_11546970635pm643380004.jpg"
        #voc_txt_path = "D:/Training/First cut model/model/Furst Cut Sub location code Model/voc-model-labels-exp.txt"        
        orig_image = read_img(img_name)
        ### model load 
        
        image_size = 300
        image_mean = np.array([127, 127, 127])  # RGB layout
        image_std = 128.0
        infer_transforms = [resize, subtract_means, misc, to_tensor]
        class_names = [name.strip() for name in open(voc_txt_path).readlines()]
    #    print("dataframe info",df.info())
        actual_label_list = []
        pred_label_list = []
        image_name_list = []
        
    #    orig_image = image.copy()
        height, width, _ = orig_image.shape
        image = get_infer_transformed(orig_image,None,None,infer_transforms)
        images = image.unsqueeze(0)
        
        ######## Forward #############
        t1 = time.time()
        i = 1
        all_out = model.forward(images)
        print (i, 'load time {}'.format(time.time()-t1))
        i = i + 1
        list_image.append(all_out)
        #print('load time {}'.format(time.time()-t1))
        
        
        ###### Final Stage Post Processing #####
        bx = all_out[:,:4] # get boxes
        lb = all_out[:,4]  # get labels
        conf = all_out[:,5] # get probabilities
        
        bx[:, 0] *= width
        bx[:, 1] *= height
        bx[:, 2] *= width
        bx[:, 3] *= height
        
        if bx[0,0].item() <= -999:
            #print(bx[0,0])
            print("no detection",s,ntpath.basename(img_name))
            s+=1
        #bx[0,0].item() != -999:
            #print("no detection")
            # If there is no detection , then create mAP data for that image
            empty_l = ["BACKGROUND",0.0,0,0,0,0]
            # with open(mAP_DATA_PATH + 'mAP_Data/prediction/{}.txt'.format(image_path.split('/')[-1].split('.jpg')[0]),'w') as f:
            #     print(*empty_l,file=f)
            pred_label_list.append('NO_DETECTION')
            #continue
        else:
        
            mAP_list = []

            v+=1
            for kj in range(bx.shape[0]):
            
                box = bx[kj,:]
                labels = lb[kj]
                probs = conf[kj]
            
                xmin,ymin,xmax,ymax = int(box[0]), int(box[1]), int(box[2]), int(box[3])
            
                pred_label = class_names[int(labels.item())]
                
                mAP_list.append([pred_label,probs.detach().item(),xmin,ymin,xmax,ymax])
            print("detect",v,ntpath.basename(img_name),mAP_list)    
            img_cv = drawBoundingBox(orig_image, mAP_list)
            jpg_name = ntpath.basename(img_name)
            new_path = os.path.join(image_new,jpg_name)
            #cv2.imwrite(r'D:\\DMI\\GM\training\\StillImg_ValidationModel\\pytorch-ssd/fender_pred\\PSIDE_1GCUYGEL4KZ110232_CHEVROLET_SILVERADO 1500_i16479_11gcuygel4kz11023220190405pside.jpg',img_cv)
            cv2.imwrite(new_path,img_cv)
            #img_show = Image.fromarray(imgcv)       
        #img_show.show()
        #cv2.imwrite("gueswho.jpg", imgcv)
        #plt.imshow(imgcv)
        #img_show = Image.open('gueswho.jpg')
        # else:
        #     print("This image is not readable:",ntpath.basename(img_name))


## Expected behavior

The model should take hardly 2-3 seconds on loading and 1s time initialization on inferences.

## Environment

Please copy and paste the output from our
[environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py)
(or fill out the checklist below manually).

You can get the script and run it with:
```
wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
# For security purposes, please check the contents of collect_env.py before running it.
python collect_env.py
```

 - PyTorch Version (e.g., 1.0): 1.0.0
 - OS (e.g., Linux): For Training used Linux 16.04 and for inferences using mobile devices with mobile cpu's
 - How you installed PyTorch (`conda`, `pip`, source): pip
 - Build command you used (if compiling from source):
 - Python version: 3.6
 - CUDA/cuDNN version: cuda9.0.176_cudnn7.4.1_1
 - GPU models and configuration: Tesla M60 ( Azure NC6 instance)
 - Any other relevant information: We have used 50k records and used SSD + mobilenetv2 model converted to torchscr .pt model file and did the inferences.

## Additional context

<!-- Add any other context about the problem here. --> We need to decrease first-time loading and initialization lag.


cc @gmagogsfm