Repository: pm4-graders/3ES
Issue category: Performance Issue

Fix CV - test on more samples and make adjustments to make it more robust



Functions to edit:
backend/app/core/cv_result.py:Exam.__init__
cv/DigitRecognizer.py:DigitRecognizer.recognize_digits_in_photo
cv/DigitRecognizer.py:DigitRecognizer.predict_double_number
cv/DigitRecognizer.py:DigitRecognizer.predict_handwritten_cell
cv/DigitRecognizer.py:DigitRecognizer.find_grid_in_image
cv/DigitRecognizer.py:DigitRecognizer.get_grid_cells
cv/DigitRecognizer.py:DigitRecognizer.get_lines

Patch:
diff --git a/.gitignore b/.gitignore
index c1a2a16..0f72b94 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,5 +5,7 @@
 /cv/testImages/kanti_img2.jpg
 /cv/testImages/kanti_img1.jpeg
 /cv/testImages/kanti_img2.jpeg
+/cv/testImages/kanti_telegram_compressed_1.jpg
+/cv/testImages/kanti_telegram_compressed_2.jpg
 /cv/__pycache__/
 /cv/Models/__pycache__/
\ No newline at end of file
diff --git a/backend/app/core/cv_result.py b/backend/app/core/cv_result.py
index 923d430..d2dd06c 100644
--- a/backend/app/core/cv_result.py
+++ b/backend/app/core/cv_result.py
@@ -12,11 +12,12 @@ def __init__(self, number, date_of_birth):
 
 
 class Exam:
-    def __init__(self, year, subject, score, confidence, exercises):
+    def __init__(self, year, subject, total_score, total_score_confidence, exercises):
         self.year = year
         self.subject = subject
-        self.score = score
-        self.confidence = confidence
+        self.total_score = total_score
+        #This is the confidence of the total score reached (how many points there are in the total cell)
+        self.total_score_confidence = total_score_confidence
         self.exercises = exercises
 
     def calc_exercises_score(self):
diff --git a/cv/DigitRecognizer.py b/cv/DigitRecognizer.py
index 9ee434b..ccceb95 100644
--- a/cv/DigitRecognizer.py
+++ b/cv/DigitRecognizer.py
@@ -11,11 +11,10 @@
 sys.path.append(sys.path[0] + '/..')
 from backend.app.core.cv_result import *
 
-
 class DigitRecognizer:
     
     global DEBUG_MODE
-    
+
     def __init__(self, debug_mode=False):
         # initialize any variables
         global DEBUG_MODE
@@ -51,15 +50,15 @@ def recognize_digits_in_photo(self, photo):
         else:
             segmentation = DocumentSegmentationCV()
 
-        #aligned_photo = segmentation.align_document(photo)
+        aligned_photo = segmentation.align_document(photo)
         
-        #if(DEBUG_MODE):
-            #self.debug_display_image('aligned',aligned_photo)
+        if(DEBUG_MODE):
+            self.debug_display_image('aligned',aligned_photo)
         
-        grid_mask, grid = self.find_grid_in_image(photo)
+        grid_mask, grid = self.find_grid_in_image(aligned_photo)
 
         if(DEBUG_MODE):
-            self.debug_display_image("grid_only", grid)
+            self.debug_display_image("grid_mask", grid_mask)
 
         grid_cells, column_count = self.get_grid_cells(grid, grid_mask)
         
@@ -82,20 +81,21 @@ def recognize_digits_in_photo(self, photo):
             result_cell = grid_cells[index + 2*column_count]
 
             if(index % column_count == 0):
-                print("First, 'Erreichte Punkte text'")
+                #print("First, 'Erreichte Punkte text'")
                 #TODO: OCR over header_cell, points_cell and result_cell and check that they say the right thing.
+                pass
 
             if(index % column_count > 0 and index % column_count < column_count-1):
-                print("Handwritten Cell")
+                #print("Handwritten Cell")
                 
-                if DEBUG_MODE:
+                if DEBUG_MODE and index == 1:
                     self.debug_display_image("cell", result_cell)
                 
                 pred_class_label, pred_confidence = self.predict_handwritten_cell(result_cell, class_labels, model)
 
                 exercises.append(Exercise(index, pred_class_label, pred_confidence, "?"))
             elif(index % column_count != 0):
-                print("Last Handwritten Cell, 'Total'")
+                #print("Last Handwritten Cell, 'Total'")
 
                 total_score, total_score_confidence = self.predict_double_number(result_cell, class_labels, model)
 
@@ -103,7 +103,7 @@ def recognize_digits_in_photo(self, photo):
 
         cv_res = CVResult(Candidate("?", "?"), exam=exam, result_validated=False)
 
-        if(exam.score == exam.calc_total_score()):
+        if(exam.total_score == exam.calc_exercises_score()):
             cv_res.result_validated = True
         return cv_res
 
@@ -125,8 +125,14 @@ def predict_double_number(self, original_cell, class_labels, model):
             widthToAdd = int((totalHeight - w) / 2)
             res = cv2.copyMakeBorder(res, 20, 20, widthToAdd, widthToAdd, cv2.BORDER_CONSTANT, value=(0,0,0))
             class_label, class_confidence = self.predict_handwritten_cell(res, class_labels, model, False)
+
+            #cv2.imshow("Handwritten Digit", res)
+            #cv2.waitKey(0)
             found_numbers.append((class_label, class_confidence))
         
+        if(found_numbers == 0):
+            return 0, 0
+
         result = ""
         confidence_sum = 0
         for number, confidence in found_numbers:
@@ -153,8 +159,8 @@ def predict_handwritten_cell(self, original_cell, class_labels, model, do_thresh
             tcell = np.expand_dims(tcell, axis=(0, -1))
             tcell = utils.normalize_images(tcell)
 
-            prediction = model.predict(tcell)
-            print(prediction)
+            prediction = model.predict(tcell, verbose=0)
+            #print(prediction)
 
             # Get the index of the highest probability
             predicted_class_index = np.argmax(prediction)
@@ -163,7 +169,7 @@ def predict_handwritten_cell(self, original_cell, class_labels, model, do_thresh
             predicted_class_label = class_labels[predicted_class_index]
 
             # Print the predicted class label
-            print("The predicted class is:", predicted_class_label)
+            #print("The predicted class is:", predicted_class_label)
 
             return int(predicted_class_label), prediction[0][predicted_class_index]
 
@@ -194,27 +200,37 @@ def find_grid_in_image(self, image):
         #The grid might be a bit warped so we want to fix this.
         out = self.fix_perspective(out, best_cnt)
 
+        #Resize the grid to scale to our wanted reference
+        width = out.shape[1]
+        wanted_width = 2000
+        scale_percent = wanted_width / width
+
+        out = cv2.resize(out, (int(out.shape[1] * scale_percent), int(out.shape[0] * scale_percent)))
+
         if(DEBUG_MODE):
-            self.debug_display_image("Grid, perspective fixed", out)
+            self.debug_display_image("Grid, perspective fixed and resized", out)
 
         #Out is already crayscale so we don't need to convert to grey but we need to blur it
-        blur = cv2.GaussianBlur(out, (5,5), 0)
+        blur = cv2.GaussianBlur(out, (7,7), 0)
 
         #Apply adaptive threshold so we have independent illumination
         thresh = cv2.adaptiveThreshold(blur, 255, 1, 1, 11, 2)
         
-        horizontal = self.get_lines(thresh.copy(), (50,1), 100)
-        vertical = self.get_lines(thresh.copy(), (1, 40), 80, False)
+        horizontal = self.get_lines(thresh.copy(), (70,1), 80)
+        vertical = self.get_lines(thresh.copy(), (1, 50), 70, False)
 
         eroded = cv2.bitwise_or(horizontal, vertical)
 
         if DEBUG_MODE:
-            self.debug_display_image("grid, before blur", eroded)
+            self.debug_display_image("grid mask, before blur", eroded)
 
         #Blur the result a little bit so the lines are more prevalent
         cv2.blur(eroded, (7,7), eroded)
         _, eroded = cv2.threshold(eroded, 100, 255, cv2.THRESH_BINARY) #we can take anything that isn't really black.
 
+        if DEBUG_MODE:
+            self.debug_display_image("grid mask, after blur", eroded)
+
         return eroded, out
 
     def get_grid_cells(self, grid, grid_mask):
@@ -230,11 +246,11 @@ def zoom_border(image, zoom):
             return image
         
         #Get vertical lines only
-        vert = self.get_lines(grid_mask.copy(), (1, 40), 80, False)
+        vert = self.get_lines(grid_mask.copy(), (1, 50), 70, False)
         
         #Zoom into the image so the outer borders are gone
         vert = zoom_border(vert, 1.1)
-        
+
         contours, _ = cv2.findContours(vert, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
         
         column_count = len(contours)+1
@@ -242,6 +258,9 @@ def zoom_border(image, zoom):
         result_cells = []
         invert = 255 - grid_mask
 
+        if(DEBUG_MODE):
+            self.debug_display_image("inverted", invert)
+
         #Find contours of inverted 
         contours, _ = cv2.findContours(invert, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
 
@@ -252,11 +271,11 @@ def zoom_border(image, zoom):
         row = []
         for (i, c) in enumerate(contours, 1):
             area = cv2.contourArea(c)
-            if area > 4000:
+            if area > 3000:
                 row.append(c)
                 if i % column_count == 0:  
-                    (contours, _) = imutils_contours.sort_contours(row, method="left-to-right")
-                    grid_rows.append(contours)
+                    (conts, _) = imutils_contours.sort_contours(row, method="left-to-right")
+                    grid_rows.append(conts)
                     row = []
 
         if len(grid_rows) != 3 or len(row) != 0:
@@ -283,8 +302,6 @@ def get_lines(self, mat, kernel, min_line_size, is_horizontal = True):
         structure = cv2.getStructuringElement(cv2.MORPH_RECT, kernel)
         mat = cv2.erode(mat, structure)
 
-        #self.debug_display_image("matuneroded",mat)
-
         #The horizontal / vertical structures have to be wide enough to be a line.
         contours, _ = cv2.findContours(mat, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
         for c in contours:
@@ -293,10 +310,19 @@ def get_lines(self, mat, kernel, min_line_size, is_horizontal = True):
                 cv2.drawContours(mat, [c], -1, (0,0,0), -1)
             if not is_horizontal and h < min_line_size:
                  cv2.drawContours(mat, [c], -1, (0,0,0), -1)
+        
+        if(DEBUG_MODE):
+            self.debug_display_image("eroded structures",mat)
+
+        mat = cv2.dilate(mat, structure, iterations=50)
 
-        #self.debug_display_image("matuneroded",mat)
+        if(DEBUG_MODE):
+            self.debug_display_image("dilated structures",mat)
 
-        mat = cv2.dilate(mat, structure, iterations=4)
+        mat = cv2.dilate(mat, cv2.getStructuringElement(cv2.MORPH_RECT, (2,2)))
+
+        if(DEBUG_MODE):
+            self.debug_display_image("'blurred', dilated structures",mat)
         return mat
 
     def find_largest_contours(self, contours):
diff --git a/cv/app.py b/cv/app.py
index 78c985c..3b6402c 100644
--- a/cv/app.py
+++ b/cv/app.py
@@ -1,13 +1,38 @@
 import cv2
 from DigitRecognizer import DigitRecognizer
+import time
 
-image = cv2.imread("cv/testImages/white_with_watermark.jpg")
-#image = cv2.imread("cv/testImages/small_nrs.png")
-#image = cv2.imread("cv/testImages/kanti_img1.jpeg")
-#image = cv2.imread("cv/testImages/kanti_img2.jpeg")
-#image = cv2.imread("cv/testImages/corner_outside_frame.jpg",1)
+images = []
 
+images.append(cv2.imread("cv/testImages/kanti_img1.jpeg"))
+images.append(cv2.imread("cv/testImages/kanti_img2.jpeg"))
+images.append(cv2.imread("cv/testImages/kanti_telegram_compressed_1.jpg"))
+images.append(cv2.imread("cv/testImages/kanti_telegram_compressed_2.jpg"))
+images.append(cv2.imread("cv/testImages/straight.jpg"))
+images.append(cv2.imread("cv/testImages/perspective.jpg"))
+images.append(cv2.imread("cv/testImages/crooked.jpg"))
+images.append(cv2.imread("cv/testImages/lighting.jpg"))
+images.append(cv2.imread("cv/testImages/mirror.jpg"))
+images.append(cv2.imread("cv/testImages/multiple.jpg"))
+images.append(cv2.imread("cv/testImages/rug.jpg"))
+images.append(cv2.imread("cv/testImages/wavy.jpg"))
+images.append(cv2.imread("cv/testImages/weird_bg.jpg"))
+images.append(cv2.imread("cv/testImages/crunched.jpg"))
 
-recognizer = DigitRecognizer()
+
+recognizer = DigitRecognizer(False)
 #cv2.imshow('image', image)
-exam = recognizer.recognize_digits_in_photo(image)
\ No newline at end of file
+for index, image in enumerate(images):
+    start = time.time()
+    try:
+        cv_result = recognizer.recognize_digits_in_photo(image)
+        print("Result validated: " + str(cv_result.result_validated))
+
+        print(', '.join([str(exercise.score) for exercise in cv_result.exam.exercises]) + " | Total: " + str(cv_result.exam.total_score))
+
+        print("IMAGE " + str(index+1) + " PASSED")
+    except:
+        print("IMAGE " + str(index+1) + " DID NOT PASS")
+        pass
+    end = time.time()
+    print(end - start)

Test patch: diff --git a/cv/testImages/crooked.jpg b/cv/testImages/crooked.jpg
new file mode 100644
index 0000000..2fade6f
Binary files /dev/null and b/cv/testImages/crooked.jpg differ
diff --git a/cv/testImages/crunched.jpg b/cv/testImages/crunched.jpg
new file mode 100644
index 0000000..3c214c5
Binary files /dev/null and b/cv/testImages/crunched.jpg differ
diff --git a/cv/testImages/lighting.jpg b/cv/testImages/lighting.jpg
new file mode 100644
index 0000000..53c97fc
Binary files /dev/null and b/cv/testImages/lighting.jpg differ
diff --git a/cv/testImages/mirror.jpg b/cv/testImages/mirror.jpg
new file mode 100644
index 0000000..c904272
Binary files /dev/null and b/cv/testImages/mirror.jpg differ
diff --git a/cv/testImages/multiple.jpg b/cv/testImages/multiple.jpg
new file mode 100644
index 0000000..bcb3a8c
Binary files /dev/null and b/cv/testImages/multiple.jpg differ
diff --git a/cv/testImages/perspective.jpg b/cv/testImages/perspective.jpg
new file mode 100644
index 0000000..01fedee
Binary files /dev/null and b/cv/testImages/perspective.jpg differ
diff --git a/cv/testImages/rug.jpg b/cv/testImages/rug.jpg
new file mode 100644
index 0000000..0ed13ad
Binary files /dev/null and b/cv/testImages/rug.jpg differ
diff --git a/cv/testImages/straight.jpg b/cv/testImages/straight.jpg
new file mode 100644
index 0000000..78b8dcf
Binary files /dev/null and b/cv/testImages/straight.jpg differ
diff --git a/cv/testImages/wavy.jpg b/cv/testImages/wavy.jpg
new file mode 100644
index 0000000..9e090c7
Binary files /dev/null and b/cv/testImages/wavy.jpg differ
diff --git a/cv/testImages/weird_bg.jpg b/cv/testImages/weird_bg.jpg
new file mode 100644
index 0000000..c9360e2
Binary files /dev/null and b/cv/testImages/weird_bg.jpg differ