added template matching

app.py

@@ -1,10 +1,11 @@
+import numpy as np
 import pandas as pd
 from PIL import Image
 import streamlit as st
 from streamlit_drawable_canvas import st_canvas
 from streamlit_image_select import image_select
 from streamlit_sortables import sort_items
-
+from py4matching import template as m
 
 def expand2square(imgpath, background_color=(0, 0, 0)):
     pil_img = Image.open(imgpath)
@@ -25,10 +26,13 @@ def expand2square(imgpath, background_color=(0, 0, 0)):
 def loading_data(files):
     imgs = []
     imgs_names = []
+    imgs_dict = {}
     for file in files:
-        imgs.append(expand2square(file))
+        image = expand2square(file)
+        imgs.append(image)
         imgs_names.append(file.name)
-    return imgs, imgs_names
+        imgs_dict[file.name] = image
+    return imgs, imgs_names, imgs_dict
 
 if 'uploaded' not in st.session_state:
     st.session_state['uploaded'] = False
@@ -47,7 +51,7 @@ else:
 if st.session_state['uploaded'] is True:
 
     # Loading uploaded images and cache the data
-    imgs, imgs_path = loading_data(images)
+    imgs, imgs_path, imgs_dict = loading_data(images)
 
     # Specify canvas parameters in application
     drawing_mode = st.sidebar.selectbox(
@@ -86,13 +90,40 @@ if st.session_state['uploaded'] is True:
         imgs_path2.pop(master_index)
         sorted_items = sort_items(imgs_path2, multi_containers=False, direction='vertical')
 
+    TEMPLATE_SIZE = 64
+    SEARCH_BUFFER = 32
+
 
     # if canvas_result.image_data is not None:
     #    st.image(canvas_result.image_data)
     if canvas_result.json_data is not None:
-        objects = pd.json_normalize(canvas_result.json_data["objects"])  # need to convert obj to str because PyArrow
-
-        for col in objects.select_dtypes(include=['object']).columns:
-            objects[col] = objects[col].astype("str")
-        st.dataframe(objects)
-
+        df = pd.json_normalize(canvas_result.json_data["objects"])  # need to convert obj to str because PyArrow
+
+        if 'type' in df.keys():
+            #st.write(df.keys())
+            df = df[['type', 'left', 'top']]
+            df[sorted_items] = np.nan
+
+            for index, row in df.iterrows():
+                #print(row['type'], row['top'])
+                i = int(row['left'])
+                j = int(row['top'])
+
+                template = np.array(imgs[master_index])[j:j+TEMPLATE_SIZE,
+                                                        i:i+TEMPLATE_SIZE]
+                # Loop on slave imgs
+                st.image(template)
+                for item in sorted_items:
+                    st.write(item)
+
+                    search_area = np.array(imgs_dict[item])[j-SEARCH_BUFFER:j+TEMPLATE_SIZE+SEARCH_BUFFER,
+                                                            i-SEARCH_BUFFER:i+TEMPLATE_SIZE+SEARCH_BUFFER]
+                    st.image(search_area)
+
+                    px, py, max_val = m.template_match(template.astype('uint8'), search_area.astype('uint8'))
+                    st.write(py)
+
+            #Covert df to string for printing
+            for col in df.select_dtypes(include=['object']).columns:
+                df[col] = df[col].astype("str")
+            st.dataframe(df)

py4matching/template.py

@@ -0,0 +1,57 @@
+import cv2
+
+def template_match(img_master, img_slave, method='cv2.TM_CCOEFF_NORMED', mlx=1, mly=1, show=True):
+    # Apply image oversampling
+
+    img_master = cv2.cvtColor(img_master, cv2.COLOR_BGR2GRAY)
+    img_slave = cv2.cvtColor(img_slave, cv2.COLOR_BGR2GRAY)
+
+    img_master = cv2.resize(img_master, None, fx=mlx, fy=mly, interpolation=cv2.INTER_CUBIC)
+    img_slave = cv2.resize(img_slave, None, fx=mlx, fy=mly, interpolation=cv2.INTER_CUBIC)
+
+    res = cv2.matchTemplate(img_slave, img_master, eval(method))
+
+    w, h = img_master.shape[::-1]
+    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
+
+    # Control if the method is TM_SQDIFF or TM_SQDIFF_NORMED, take minimum value
+    if method in [cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED]:
+        top_left = min_loc
+    else:
+        top_left = max_loc
+    bottom_right = (top_left[0] + w, top_left[1] + h)
+
+    # Retrieve center coordinates
+    px = (top_left[0] + bottom_right[0]) / (2.0 * mlx)
+    py = (top_left[1] + bottom_right[1]) / (2.0 * mly)
+
+    # # Scale images for visualization
+    # img_master_scaled = cv2.convertScaleAbs(img_master, alpha=(255.0 / 500))
+    # img_slave_scaled = cv2.convertScaleAbs(img_slave, alpha=(255.0 / 500))
+    #
+    # cv2.rectangle(img_slave_scaled, top_left, bottom_right, 255, 2 * mlx)
+    #
+    # if show == True:
+    #     plt.figure(figsize=(20, 10))
+    #     plt.subplot(131), plt.imshow(res, cmap='gray')
+    #     plt.title('Matching Result'), plt.xticks([]), plt.yticks([])
+    #     plt.subplot(132), plt.imshow(img_master_scaled, cmap='gray')
+    #     plt.title('Detected Point'), plt.xticks([]), plt.yticks([])
+    #     plt.subplot(133), plt.imshow(img_slave_scaled, cmap='gray')
+    #     plt.suptitle(method)
+    #     plt.show()
+
+    return px, py, max_val
+
+# import numpy as np
+# example = np.arange(2000).reshape((100, 20))
+# a_kernel = np.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]])
+# def sliding_window(data, win_shape, **kwargs):
+#         assert data.ndim == len(win_shape)
+#         shape = tuple(dn - wn + 1 for dn, wn in zip(data.shape, win_shape)) + win_shape
+#         strides = data.strides * 2
+#         return np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides, **kwargs)
+# def arrays_from_kernel(a, a_kernel):
+#         windows = sliding_window(a, a_kernel.shape)
+#         return np.where(a_kernel, windows, 0)
+# sub_arrays = arrays_from_kernel(example, a_kernel)