add: keypoints

README.md

@@ -30,7 +30,7 @@ Here is a demo of the tool:
 
 The tool currently supports various popular image matching algorithms, namely:
 - [x] [LightGlue](https://github.com/cvg/LightGlue), ICCV 2023
-- [x] [DeDoDe](https://github.com/Parskatt/DeDoDe), TBD
+- [x] [DeDoDe](https://github.com/Parskatt/DeDoDe), ArXiv 2023
 - [x] [DarkFeat](https://github.com/THU-LYJ-Lab/DarkFeat), AAAI 2023
 - [ ] [ASTR](https://github.com/ASTR2023/ASTR), CVPR 2023
 - [ ] [SEM](https://github.com/SEM2023/SEM), CVPR 2023
@@ -61,7 +61,13 @@ The tool currently supports various popular image matching algorithms, namely:
 
 ## How to use
 
-### requirements
+### HuggingFace 
+
+Just try it on HF <a href='https://huggingface.co/spaces/Realcat/image-matching-webui'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue'>  [![Open in OpenXLab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/Realcat/image-matching-webui)
+
+or deploy it locally following the instructions below.
+
+### Requirements
 ``` bash
 git clone --recursive https://github.com/Vincentqyw/image-matching-webui.git
 cd image-matching-webui
@@ -88,7 +94,8 @@ External contributions are very much welcome. Please follow the [PEP8 style guid
 - [x] add webcam support
 - [x] add [line feature matching](https://github.com/Vincentqyw/LineSegmentsDetection) algorithms
 - [x] example to add a new feature extractor / matcher
-- [ ] ransac to filter outliers
+- [x] ransac to filter outliers
+- [ ] add [rotation images](https://github.com/pidahbus/deep-image-orientation-angle-detection) options before matching 
 - [ ] support export matches to colmap ([#issue 6](https://github.com/Vincentqyw/image-matching-webui/issues/6))
 - [ ] add config file to set default parameters
 - [ ] dynamically load models and reduce GPU overload

app.py

@@ -28,7 +28,7 @@ def ui_reset_state(
     extract_max_keypoints,
     keypoint_threshold,
     key,
-    enable_ransac=False,
+    # enable_ransac=False,
     ransac_method="RANSAC",
     ransac_reproj_threshold=8,
     ransac_confidence=0.999,
@@ -41,7 +41,7 @@ def ui_reset_state(
     key = list(matcher_zoo.keys())[0]
     image0 = None
     image1 = None
-    enable_ransac = False
+    # enable_ransac = False
     return (
         image0,
         image1,
@@ -52,12 +52,14 @@ def ui_reset_state(
         ui_change_imagebox("upload"),
         ui_change_imagebox("upload"),
         "upload",
-        None,
+        None,  # keypoints
+        None,  # raw matches
+        None,  # ransac matches
         {},
         {},
         None,
         {},
-        False,
+        # False,
         "RANSAC",
         8,
         0.999,
@@ -145,7 +147,7 @@ def run(config):
                         # )
                     with gr.Accordion("RANSAC Setting", open=True):
                         with gr.Row(equal_height=False):
-                            enable_ransac = gr.Checkbox(label="Enable RANSAC")
+                            # enable_ransac = gr.Checkbox(label="Enable RANSAC")
                             ransac_method = gr.Dropdown(
                                 choices=ransac_zoo.keys(),
                                 value="RANSAC",
@@ -192,7 +194,7 @@ def run(config):
                     match_setting_max_features,
                     detect_keypoints_threshold,
                     matcher_list,
-                    enable_ransac,
+                    # enable_ransac,
                     ransac_method,
                     ransac_reproj_threshold,
                     ransac_confidence,
@@ -223,18 +225,23 @@ def run(config):
                     )
 
             with gr.Column():
-                output_mkpts = gr.Image(
-                    label="Keypoints Matching", type="numpy"
+                output_keypoints = gr.Image(label="Keypoints", type="numpy")
+                output_matches_raw = gr.Image(label="Raw Matches", type="numpy")
+                output_matches_ransac = gr.Image(
+                    label="Ransac Matches", type="numpy"
                 )
                 with gr.Accordion(
-                    "Open for More: Matches Statistics", open=True
+                    "Open for More: Matches Statistics", open=False
                 ):
                     matches_result_info = gr.JSON(label="Matches Statistics")
                     matcher_info = gr.JSON(label="Match info")
 
-                with gr.Accordion("Open for More: Geometry info", open=False):
-                    output_wrapped = gr.Image(label="Wrapped Pair", type="numpy")
-                    geometry_result = gr.JSON(label="Reconstructed Geometry")
+                with gr.Accordion("Open for More: Warped Image", open=False):
+                    output_wrapped = gr.Image(
+                        label="Wrapped Pair", type="numpy"
+                    )
+                    with gr.Accordion("Open for More: Geometry info", open=False):
+                        geometry_result = gr.JSON(label="Reconstructed Geometry")
 
             # callbacks
             match_image_src.change(
@@ -250,7 +257,9 @@ def run(config):
 
             # collect outputs
             outputs = [
-                output_mkpts,
+                output_keypoints,
+                output_matches_raw,
+                output_matches_ransac,
                 matches_result_info,
                 matcher_info,
                 geometry_result,
@@ -270,12 +279,14 @@ def run(config):
                 input_image0,
                 input_image1,
                 match_image_src,
-                output_mkpts,
+                output_keypoints,
+                output_matches_raw,
+                output_matches_ransac,
                 matches_result_info,
                 matcher_info,
                 output_wrapped,
                 geometry_result,
-                enable_ransac,
+                # enable_ransac,
                 ransac_method,
                 ransac_reproj_threshold,
                 ransac_confidence,

common/utils.py

@@ -8,6 +8,7 @@ import gradio as gr
 from hloc import matchers, extractors
 from hloc.utils.base_model import dynamic_load
 from hloc import match_dense, match_features, extract_features
+from hloc.utils.viz import add_text, plot_keypoints
 from .viz import draw_matches, fig2im, plot_images, plot_color_line_matches
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -68,7 +69,7 @@ def gen_examples():
                 match_setting_max_features,
                 detect_keypoints_threshold,
                 mt,
-                enable_ransac,
+                # enable_ransac,
                 ransac_method,
                 ransac_reproj_threshold,
                 ransac_confidence,
@@ -105,6 +106,9 @@ def filter_matches(
         return pred
     if ransac_method not in ransac_zoo.keys():
         ransac_method = "RANSAC"
+
+    if len(mkpts0) < 4:
+        return pred
     H, mask = cv2.findHomography(
         mkpts0,
         mkpts1,
@@ -236,7 +240,7 @@ def change_estimate_geom(input_image0, input_image1, matches_info, choice):
         return None, None
 
 
-def display_matches(pred: dict):
+def display_matches(pred: dict, titles=[], dpi=300):
     img0 = pred["image0_orig"]
     img1 = pred["image1_orig"]
 
@@ -255,11 +259,8 @@ def display_matches(pred: dict):
             img0,
             img1,
             mconf,
-            dpi=300,
-            titles=[
-                "Image 0 - matched keypoints",
-                "Image 1 - matched keypoints",
-            ],
+            dpi=dpi,
+            titles=titles,
         )
         fig = fig_mkpts
     if "line0_orig" in pred.keys() and "line1_orig" in pred.keys():
@@ -302,7 +303,7 @@ def run_matching(
     extract_max_keypoints,
     keypoint_threshold,
     key,
-    enable_ransac=False,
+    # enable_ransac=False,
     ransac_method="RANSAC",
     ransac_reproj_threshold=8,
     ransac_confidence=0.999,
@@ -312,6 +313,10 @@ def run_matching(
     # image0 and image1 is RGB mode
     if image0 is None or image1 is None:
         raise gr.Error("Error: No images found! Please upload two images.")
+    # init output
+    output_keypoints = None
+    output_matches_raw = None
+    output_matches_ransac = None
 
     model = matcher_zoo[key]
     match_conf = model["config"]
@@ -341,16 +346,48 @@ def run_matching(
         pred = match_features.match_images(matcher, pred0, pred1)
         del extractor
 
-    if enable_ransac:
-        filter_matches(
-            pred,
-            ransac_method=ransac_method,
-            ransac_reproj_threshold=ransac_reproj_threshold,
-            ransac_confidence=ransac_confidence,
-            ransac_max_iter=ransac_max_iter,
-        )
+    # plot images with keypoints
+    titles = [
+        "Image 0 - Keypoints",
+        "Image 1 - Keypoints",
+    ]
+    output_keypoints = plot_images([image0, image1], titles=titles, dpi=300)
+    plot_keypoints([pred["keypoints0"], pred["keypoints1"]])
+    text = (
+        f"# keypoints0: {len(pred['keypoints0'])} \n"
+        + f"# keypoints1: {len(pred['keypoints1'])}"
+    )
+
+    add_text(0, text, fs=15)
+    output_keypoints = fig2im(output_keypoints)
+
+    # plot images with raw matches
+    titles = [
+        "Image 0 - Raw matched keypoints",
+        "Image 1 - Raw matched keypoints",
+    ]
+
+    output_matches_raw, num_matches_raw = display_matches(pred, titles=titles)
 
-    fig, num_inliers = display_matches(pred)
+    # if enable_ransac:
+    filter_matches(
+        pred,
+        ransac_method=ransac_method,
+        ransac_reproj_threshold=ransac_reproj_threshold,
+        ransac_confidence=ransac_confidence,
+        ransac_max_iter=ransac_max_iter,
+    )
+
+    # plot images with ransac matches
+    titles = [
+        "Image 0 - Ransac matched keypoints",
+        "Image 1 - Ransac matched keypoints",
+    ]
+    output_matches_ransac, num_matches_ransac = display_matches(
+        pred, titles=titles
+    )
+
+    # plot wrapped images
     geom_info = compute_geom(pred)
     output_wrapped, _ = change_estimate_geom(
         pred["image0_orig"],
@@ -358,10 +395,17 @@ def run_matching(
         {"geom_info": geom_info},
         choice_estimate_geom,
     )
+
     del pred
+
     return (
-        fig,
-        {"matches number": num_inliers},
+        output_keypoints,
+        output_matches_raw,
+        output_matches_ransac,
+        {
+            "number raw matches": num_matches_raw,
+            "number ransac matches": num_matches_ransac,
+        },
         {
             "match_conf": match_conf,
             "extractor_conf": extract_conf,

hloc/match_dense.py

@@ -340,8 +340,8 @@ def match_images(model, image_0, image_1, conf, device="cpu"):
             "image1": image1.squeeze().cpu().numpy(),
             "image0_orig": image_0,
             "image1_orig": image_1,
-            "keypoints0": kpts0.cpu().numpy(),
-            "keypoints1": kpts1.cpu().numpy(),
+            "keypoints0": kpts0_origin.cpu().numpy(),
+            "keypoints1": kpts1_origin.cpu().numpy(),
             "keypoints0_orig": kpts0_origin.cpu().numpy(),
             "keypoints1_orig": kpts1_origin.cpu().numpy(),
             "original_size0": np.array(image_0.shape[:2][::-1]),

hloc/match_features.py

@@ -369,15 +369,19 @@ def match_images(model, feat0, feat1):
     # rescale the keypoints to their original size
     s0 = feat0["original_size"] / feat0["size"]
     s1 = feat1["original_size"] / feat1["size"]
-    kpts0_origin = scale_keypoints(torch.from_numpy(mkpts0 + 0.5), s0) - 0.5
-    kpts1_origin = scale_keypoints(torch.from_numpy(mkpts1 + 0.5), s1) - 0.5
+    kpts0_origin = scale_keypoints(torch.from_numpy(kpts0 + 0.5), s0) - 0.5
+    kpts1_origin = scale_keypoints(torch.from_numpy(kpts1 + 0.5), s1) - 0.5
+
+    mkpts0_origin = scale_keypoints(torch.from_numpy(mkpts0 + 0.5), s0) - 0.5
+    mkpts1_origin = scale_keypoints(torch.from_numpy(mkpts1 + 0.5), s1) - 0.5
+
     ret = {
         "image0_orig": feat0["image_orig"],
         "image1_orig": feat1["image_orig"],
-        "keypoints0": kpts0,
-        "keypoints1": kpts1,
-        "keypoints0_orig": kpts0_origin.numpy(),
-        "keypoints1_orig": kpts1_origin.numpy(),
+        "keypoints0": kpts0_origin.numpy(),
+        "keypoints1": kpts1_origin.numpy(),
+        "keypoints0_orig": mkpts0_origin.numpy(),
+        "keypoints1_orig": mkpts1_origin.numpy(),
         "mconf": mconfid,
     }
     del feat0, feat1, desc0, desc1, kpts0, kpts1, kpts0_origin, kpts1_origin