Webcam and image classifier integration

Hafizur_Rahman: <p>Hello, I am using a code to classify real time  image, But I am confused about it. Could you please take  a look on the code and help me to figure out any error:</p>
<pre><code class="lang-auto">import numpy as np  
import torch
import torch.nn as nn
import torchvision 
from torch.autograd import Variable
from torchvision import transforms
import PIL 
import cv2

#This is the Label
Labels = ['Perfect','Defected']

# Let's preprocess the inputted frame

data_transforms = torchvision.transforms.Compose([
    torchvision.transforms.Resize(size=(352, 288)),
    torchvision.transforms.RandomHorizontalFlip(),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])



device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Load the model and set in eval
resnet18 = torchvision.models.resnet18(pretrained=True)
resnet18.fc = torch.nn.Sequential(nn.Linear(resnet18.fc.in_features,512),
                                  nn.ReLU(),
                                  nn.Dropout(),
                                  nn.Linear(512, 2))

resnet18.load_state_dict(torch.load('defect_classifier_demo.pt', map_location=torch.device('cpu')))
resnet18.eval()



#Set the Webcam 
def Webcam_720p():
    cap.set(3,1280)
    cap.set(4,720)







def preprocess(image):
    image = PIL.Image.fromarray(image) #Webcam frames are numpy array format
                                       #Therefore transform back to PIL image
    print(image)                             
    image = data_transforms(image)
    image = image.float()
    #image = Variable(image, requires_autograd=True)
    image = image.cpu()
    image = image.unsqueeze(0) #I don't know for sure but Resnet-50 model seems to only
                               #accpets 4-D Vector Tensor so we need to squeeze another
    return image                            #dimension out of our 3-D vector Tensor
    
    
#Let's start the real-time classification process!
                                  
cap = cv2.VideoCapture(0) #Set the webcam
Webcam_720p()

fps = 0
show_score = 0
show_res = 'Nothing'
sequence = 0

while True:
    ret, frame = cap.read() #Capture each frame
    
    
    if fps == 4:
        image        = frame[100:450,150:570]
        image_data   = preprocess(image)
        #print(image_data)
        prediction   = resnet18(image_data)[0]
        torch.nn.functional.softmax(prediction, dim=0)
        #result,score = argmax(prediction)
        prediction = prediction.cpu().detach().numpy()
        predicted_class_index = np.argmax(prediction)
        predicted_class_name = Labels[predicted_class_index]
        print(prediction)
        print(predicted_class_index)
        print(predicted_class_name)
        #print(result)
        #print(score)
        #prediction = np.vectorize(prediction)
        fps = 0
        if prediction.any():
            show_res  = predicted_class_name
            show_score= prediction
        else:
            show_res   = "Nothing"
            show_score = prediction
        
    fps += 1
    cv2.putText(frame, '%s' %(show_res),(950,250), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 3)
    #cv2.putText(frame, '(score = %.5f)' %(show_score), (950,300), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),2)
    cv2.rectangle(frame,(400,150),(900,550), (250,0,0), 2)
    cv2.imshow("ASL SIGN DETECTER", frame)

    if cv2.waitKey(1) &amp; 0xFF == ord('q'):
        break

cap.release()
cv2.destroyWindow("ASL SIGN DETECTER") ```</code></pre>

ptrblck: <p>What kind of error are you seeing? Could you post the error message with the complete stack trace, please?</p>

Hafizur_Rahman: <p>The script is working wired. <div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/8/c/8c244ced0de91ef570d2043f71b1b611afd4e78e.jpeg" data-download-href="https://discuss.pytorch.org/uploads/default/8c244ced0de91ef570d2043f71b1b611afd4e78e" title="Screenshot 2021-02-09 124435"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/8/c/8c244ced0de91ef570d2043f71b1b611afd4e78e_2_690x398.jpeg" alt="Screenshot 2021-02-09 124435" data-base62-sha1="jZKCoCAvY59BhdEAwQlnLXEXinI" width="690" height="398" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/8/c/8c244ced0de91ef570d2043f71b1b611afd4e78e_2_690x398.jpeg, https://discuss.pytorch.org/uploads/default/original/3X/8/c/8c244ced0de91ef570d2043f71b1b611afd4e78e.jpeg 2x" data-dominant-color="A67D71"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">Screenshot 2021-02-09 124435</span><span class="informations">1285×742 260 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div> <div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/8/f/8fdcc872266a4cb645f3fcfc04c0594cf5f00d8a.jpeg" data-download-href="https://discuss.pytorch.org/uploads/default/8fdcc872266a4cb645f3fcfc04c0594cf5f00d8a" title="Screenshot 2021-02-09 124420"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/8/f/8fdcc872266a4cb645f3fcfc04c0594cf5f00d8a_2_690x402.jpeg" alt="Screenshot 2021-02-09 124420" data-base62-sha1="kwFiPeUes82C69pyzbvU1uFe2Vk" width="690" height="402" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/8/f/8fdcc872266a4cb645f3fcfc04c0594cf5f00d8a_2_690x402.jpeg, https://discuss.pytorch.org/uploads/default/optimized/3X/8/f/8fdcc872266a4cb645f3fcfc04c0594cf5f00d8a_2_1035x603.jpeg 1.5x, https://discuss.pytorch.org/uploads/default/original/3X/8/f/8fdcc872266a4cb645f3fcfc04c0594cf5f00d8a.jpeg 2x" data-dominant-color="A67D72"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">Screenshot 2021-02-09 124420</span><span class="informations">1286×751 260 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<p>When I am putting an object to predict, the prediction is changing oftenly.</p>

ptrblck: <p>Your model might be performing poorly on this data.<br>
Have you checked the validation and final test dataset accuracy and were you using images captured by the same device?<br>
If so, what was the overall performance of this model?</p>

Hafizur_Rahman: <p>Validation Accuracy is 92% and Test Accuracy is 91%.<br>
I am using same device and background also.</p>
<p>Training Statistics is like:</p>
<pre><code class="lang-auto">Epoch: 0, Training Loss: 0.67, Validation Loss: 0.38, accuracy = 0.86
Epoch: 1, Training Loss: 0.36, Validation Loss: 0.39, accuracy = 0.84
Epoch: 2, Training Loss: 0.28, Validation Loss: 0.22, accuracy = 0.91
Epoch: 3, Training Loss: 0.25, Validation Loss: 0.18, accuracy = 0.94
Epoch: 4, Training Loss: 0.21, Validation Loss: 0.25, accuracy = 0.89
Epoch: 5, Training Loss: 0.20, Validation Loss: 0.22, accuracy = 0.89
Epoch: 6, Training Loss: 0.20, Validation Loss: 0.19, accuracy = 0.93
Epoch: 7, Training Loss: 0.19, Validation Loss: 0.18, accuracy = 0.93
Epoch: 8, Training Loss: 0.16, Validation Loss: 0.24, accuracy = 0.91
Epoch: 9, Training Loss: 0.16, Validation Loss: 0.24, accuracy = 0.92
</code></pre>

ptrblck: <p>The test accuracy seems to be quite high. How are these “bad results” created? Are you using another dataset for it?</p>

Hoang_Tr_n: <p>Hey guys i’m trying to deploy my fruits recognition model on webcam and this is my error that i’m facing with<br>
Traceback (most recent call last):<br>
File “c:/Users/Admin/Documents/AI-PYTHON/TEST/realtime_recognization.py”, line 60, in <br>
prediction   = model(image_data)<br>
File “C:\Users\Admin\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py”, line 550, in <strong>call</strong><br>
result = self.forward(*input, **kwargs)<br>
File “c:\Users\Admin\Documents\AI-PYTHON\TEST\training_model.py”, line 41, in forward<br>
return self.network(xb)<br>
File “C:\Users\Admin\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py”, line 550, in <strong>call</strong><br>
result = self.forward(*input, **kwargs)<br>
File “C:\Users\Admin\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\container.py”, line 100, in forward<br>
input = module(input)<br>
File “C:\Users\Admin\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py”, line 550, in <strong>call</strong><br>
result = self.forward(*input, **kwargs)<br>
File “C:\Users\Admin\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\linear.py”, line 87, in forward<br>
return F.linear(input, self.weight, self.bias)<br>
File “C:\Users\Admin\anaconda3\envs\py36\lib\site-packages\torch\nn\functional.py”, line 1610, in linear<br>
ret = torch.addmm(bias, input, weight.t())<br>
RuntimeError: size mismatch, m1: [1 x 45000], m2: [36000 x 1000] at C:/cb/pytorch_1000000000000/work/aten/src\THC/generic/THCTensorMathBlas.cu:283</p>
<p>And this is my code</p>
<p>from torch import mode</p>
<p>from lib import *</p>
<p>from utils import *</p>
<p>from training_model import CnnModel</p>
<p>from PIL import Image</p>
<p>from config import resize, mean, std, save_path</p>
<p>data_transforms = transforms.Compose(</p>
<pre><code>[

    transforms.Resize(resize),

    transforms.ToTensor(),

    transforms.Normalize(mean,std)

]
</code></pre>
<p>)</p>
<p>device = get_default_device()</p>
<p>model = to_device(CnnModel(), device)</p>
<p>model = load_model(model, save_path)</p>
<p>model.eval()</p>
<p>def Webcam_720p():</p>
<pre><code>cap.set(3,1280)

cap.set(4,720)
</code></pre>
<p>def predict_image(prediction):</p>
<pre><code>prediction = prediction.cuda()

prediction = prediction.detach().numpy()

_, preds  = torch.max(prediction, dim=1)

score = np.amax(prediction)

score = '{:6f}'.format(score)

result = datasett.classes[preds[0].item()]

return result,score
</code></pre>
<p>def preprocess(image):</p>
<pre><code>image = Image.fromarray(image) #                          

image = data_transforms(image)

image = image.float()

image = image.cuda()

image = image.unsqueeze(0) 

return image 
</code></pre>
<p>cap = cv2.VideoCapture(0)</p>
<p>Webcam_720p()</p>
<p>fps = 0</p>
<p>show_score = 0</p>
<p>show_res = ‘Nothing’</p>
<p>sequence = 0</p>
<p>while True:</p>
<pre><code>ret, frame = cap.read()

    

if fps == 4:

    image        = frame[100:450,150:570]

    image_data   = preprocess(image)

    prediction   = model(image_data)

    result,score = predict_image(prediction)

    fps = 0

    if result &gt;= 0.5:

        show_res  = result

        show_score= score

    else:

        show_res   = "Nothing"

        show_score = score

    

fps += 1

cv2.putText(frame, '%s' %(show_res),(950,250), cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 3)

cv2.putText(frame, '(score = %.5f)' %(show_score), (950,300), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255),2)

cv2.rectangle(frame,(400,150),(900,550), (0,250,0), 2)

cv2.imshow("ASL SIGN DETECTER", frame)

if cv2.waitKey(1) &amp; 0xFF == ord('q'):

    break
</code></pre>
<p>cap.release()</p>
<p>cv2.destroyWindow(“ASL SIGN DETECTER”)</p>

ptrblck: <p>Based on the error message you are running into a shape mismatch in a linear layer, which is often caused by changing the input shape without using an adaptive pooling layer.<br>
In case your model was working before without the webcam images, you should check the spatial shapes of the training and webcam images and make sure they are equal.</p>

Hoang_Tr_n: <p>Yes sir you’re right , my model totally can make predictions from JPEG images but webcam images is a different thing . I wonder these mismatch errors come from this line “image        = frame[100:450,150:570]” , doesn’t it ? And i dont know what exactly it is. Could you explain it for me ?</p>

ptrblck: <p>Yes, I think the slicing operation could create an unexpected shape.<br>
The posted line of code sliced the <code>frame</code> in two dimensions using the specified start and end indices.<br>
As described before, check which shape the training images had and make sure the new images have the same.</p>