diff --git a/README.md b/README.md index fdff0e78af49c208afd15f70185f555d52802c70..55715e079853f433feb99d04e3ff18906313eb29 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Here is a demo of the tool: ![demo](assets/demo.gif) The tool currently supports various popular image matching algorithms, namely: +- [x] [DUSt3R](https://github.com/naver/dust3r), CVPR 2024 - [x] [OmniGlue](https://github.com/Vincentqyw/omniglue-onnx), CVPR 2024 - [x] [XFeat](https://github.com/verlab/accelerated_features), CVPR 2024 - [x] [RoMa](https://github.com/Vincentqyw/RoMa), CVPR 2024 diff --git a/common/config.yaml b/common/config.yaml index 5fe02682a153673cfe852f602efafdd8247d4893..f4f435df02e5b7ab79e5862d30c50d193d2fb8bf 100644 --- a/common/config.yaml +++ b/common/config.yaml @@ -29,7 +29,7 @@ matcher_zoo: display: true DUSt3R: # TODO: duster is under development - enable: false + enable: true matcher: duster dense: true info: diff --git a/common/utils.py b/common/utils.py index d961984d87d1f9940e075e5571031ff4632176bb..a5dd496686cd4fb3ea55c005950b931af50b787a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -232,7 +232,7 @@ def gen_examples(): return [pairs[i] for i in selected] # rotated examples - def gen_rot_image_pairs(count: int = 10): + def gen_rot_image_pairs(count: int = 5): path = ROOT / "datasets/sacre_coeur/mapping" path_rot = ROOT / "datasets/sacre_coeur/mapping_rot" rot_list = [45, 180, 90, 225, 270] @@ -253,6 +253,27 @@ def gen_examples(): selected = random.sample(range(len(pairs)), count) return [pairs[i] for i in selected] + def gen_scale_image_pairs(count: int = 5): + path = ROOT / "datasets/sacre_coeur/mapping" + path_scale = ROOT / "datasets/sacre_coeur/mapping_scale" + scale_list = [0.3, 0.5] + pairs = [] + for file in os.listdir(path): + if file.lower().endswith((".jpg", ".jpeg", ".png")): + for scale in scale_list: + file_scale = "{}_scale{}.jpg".format(Path(file).stem, scale) + if (path_scale / file_scale).exists(): + pairs.append( + [ + path / file, + path_scale / file_scale, + ] + ) + if len(pairs) < count: + count = len(pairs) + selected = random.sample(range(len(pairs)), count) + return [pairs[i] for i in selected] + # extramely hard examples def gen_image_pairs_wxbs(count: int = None): prefix = "datasets/wxbs_benchmark/.WxBS/v1.1" @@ -275,6 +296,7 @@ def gen_examples(): # image pair path pairs = gen_images_pairs() pairs += gen_rot_image_pairs() + pairs += gen_scale_image_pairs() pairs += gen_image_pairs_wxbs() match_setting_threshold = DEFAULT_SETTING_THRESHOLD @@ -1026,5 +1048,8 @@ def scale_image(input_path, scale_factor, output_path): width, height = img.size new_width = int(width * scale_factor) new_height = int(height * scale_factor) + new_img = Image.new("RGB", (width, height), (0, 0, 0)) img_resized = img.resize((new_width, new_height)) - img_resized.save(output_path) + position = ((width - new_width) // 2, (height - new_height) // 2) + new_img.paste(img_resized, position) + new_img.save(output_path) diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d27fe8d6f813e1525d581e2665feff4704710e98 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65632a80a46b26408c91314f2c78431be3df41d8c740e1b1b2614c98f17091a0 +size 21890 diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..26c82557bab1bb66463603bcbb23fb179ab1456a --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b95394b52baf736f467b28633536223abaf34b4219fe9c9541cd902c47539de7 +size 40252 diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4e76cf942182757709234e6b44fdef9ea255035f --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de85408453e630cc7d28cf91df73499e8c98caffef40b1b196ef019d59f09486 +size 52742 diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e87b7854fded8fa9bd423e47b93da3e4e76414d --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35bc6e9a0c6b570e9d19c91d124422f17e26012908ee089eede78fd33ce7c051 +size 65170 diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4cdcbbf48d27668ecc6e2c4ddf109a6cf1405e69 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6120fd5688d41a61a6d00605b2355a1775e0dba3b3ade8b5ad32bae35de897f8 +size 80670 diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e60569c28a50e261f661a20816fe5ab4dd265754 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea90074a3c2680086fad36ea3a875168af69d56b055a1166940fbd299abb8e9f +size 96216 diff --git a/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..022cf34ec13b5accbfcaf5c6ac9adc9ed63f4387 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/02928139_3448003521_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4371f89589919aa8bc7f4f093bf88a87994c45de67a471b5b1a6412eaa8e54e4 +size 112868 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ae239cdbed0fa5094181cfa85977a4cc5dd401b1 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:394013dede3a72016cfefb51dd45b84439523d427de47ef43f45bbd2ce07d1b1 +size 18346 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7816ec7253dac16d13da7ab2bb05eb918cf90ec9 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd82ebb523f88dddb3cfcdeb20f149cb18a70062cd67eb394b6442a8f39ec9b1 +size 29287 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..75421d467f758d274c43191085d6b99ec6987741 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8927ec9505d0043ae8030752132cf47253ebc034d93394e4b972961a5307f151 +size 37883 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4b77784210fb86c4db0f587c70061dff44d72e06 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f0c0465ba88b9296383ed21e9b4fa4116d145fae52913706a134a58247488b +size 44461 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..13b6b9d3b1d66e9356a0b97567f221882ea9bc7e --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1344408ec4e79932b9390a3340d2241eedc535c67a2811bda7390a64ddadaa75 +size 52812 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c1a7d765ecf17c513f6c716b8ba1b9421d8ad5c1 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4b253faf50d53822fdca31acfd269c5fd3df28ef6bd8182a753ca0ff626172 +size 63930 diff --git a/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..735e7c4475c84772447f96b0ca40703482044e94 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/03903474_1471484089_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0822b92d430c4d42076b0469a74e8cc6b379657b9b310b5f7674c6a47af6e8f +size 73652 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..06204bf54d2df8157c21f061496e644b6fa47b23 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bcef8bb651e67a969b5b17f55dd9b0827aaa69f42efe0da096787cae0ebc51e +size 19404 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d16aaf7bdbc453b42fc2d8c72fd9cf90bd5e3317 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdebf79043e7d889ecebf7f975d57d1202f49977dff193b158497ebcfc809a8f +size 33947 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c24e10ded3cca68b8851073f8eb19f36ee07acc2 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89bf1005e2bb111bf8bd9914bff13eb3252bf1427426bc2925c4f618bd8428b6 +size 44233 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cd4f55bd2b3587f56a8c60ab0f4c6c0298d9b197 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b69b20d8695366ccbf25d8f40c7d722e4d3a48ecb99163f46c7d9684d7e7b2e5 +size 55447 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..da7c59860bdb2812ea138b55965ebc0711c80f5f --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f92f6225593780ad721b1c14dcb8772146433b5dbf7f0b927a1e2ac86bb8c4bd +size 65029 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..807d37f1572761bf8d876384e33577f26ca257b4 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f91146bc682fc502cbefd103e44e68a6865fbf69f28946de2f63e14aed277eb5 +size 78525 diff --git a/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2c8384e6688273d4f1f31a80a3ef47a4e89a21b8 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/10265353_3838484249_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406f548692e16ac5449d1353d1b73b396285d5af82928e271186ac4692fa6b61 +size 92311 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..52b5e6a247240106fd5e5c886b7a22cf7de939c3 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f1667f20a37d2ebc6ecf94c418078492ab465277a97c313317ff01bbafad6ae +size 19279 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1ecf2d221eb38ae1bebfce5e722f0999325ac519 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:107c4cc346f8b1f992e369aedf3abdafea4698539194f40eade5b5776eef5dd5 +size 36526 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6976776119ba2f20a89e7cee4e04fdea6894677a --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:407f9290be844f5831a52b926b618eff02a4db7bc6597f063051eeeaf39d151b +size 47460 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bcd235542d03cdbb3480a44ee181c01ca48a6558 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:967e490e89b602ecaf87863d49d590da4536c10059537aaec2d43f4038e3d771 +size 62509 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b987b854cdb4d833cf6aa74df88927a975c44362 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5afc12af54be3cb0dba0922a7f566ee71135cea8369b2e4842e7ae42fe64b53 +size 76306 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..877bb2c923b487518c640125c5197b0edf0e0bfb --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f266200d9a1f2e9aefb1e3713aa70ccd0eed2d5c5af996aef120dcf18b0780 +size 91762 diff --git a/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4d0c041790cf9f3de32e608c04e589917d546556 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/17295357_9106075285_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12aaae2e7a12b49ec263d9d2a29ff730ff90fbed1d0c5ce4789550695a891cbe +size 107595 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5f9313cfc25615fe53efa05ffcdfe54b62ccdcdb --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e8dc5991d5af4b134fe7534c596aad5b88265981bba3d90156fff2fa3b7bd8 +size 19569 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e50f64360fe0805698b11202f12caaf6f939049c --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad8c1fc561fc5967d406c8ca8d0cce65344838edc46cc72e85515732a1d1a266 +size 34434 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8ad560b70e0923b134bdf4fd35c75036b9c1feb6 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:132a899e4725b23eb0d0740bfda5528e68fdfcafef932b6f48b1a47e1d906bdb +size 44062 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7713f99755d6fb12f24d59ace7c6bb30af8d0140 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ca571a824efff8b35a3aa234756d652aa9a54c33dfd98c2c5854eeb8bc2987 +size 56056 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cd312a40e414dabc167d982e7283703e8d533bae --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66177f53a20ae430a858f7aeb2049049da83b910c277b5344113508a41859179 +size 65465 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f40be8bd5671519bfa985a592d4b4a2984582d4d --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6db2f05714a37dfd8ab21d52d05afd990b47271c1469e0c965d35dcd81e2c39 +size 77708 diff --git a/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9ecb09015abbcbf32907ff8584f9427161e3ed64 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/32809961_8274055477_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3a58d711cfe35f937f5b9e9914c7dcb427065d685a816c4dc5da610403d981 +size 92547 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e3cd992394f9ace830fe1df7911f04c80e1abe0a --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bea2d09de8c92e49916c72f57ef8d0bc39fb63b51cdd2568706cbe01028a4480 +size 18119 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cf490019463c6e01cc23fc2962c468c633a53aaa --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0312626c40b38df7b64471873e2efaebc90a5a34cfe9778da7b7a9a6793975be +size 28961 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f7ac21a0279982bb6f0a8ddf715f794b1577fc71 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ab9561483b9b02fd62304ce34c337107494050635e8a16bbf4b6426f602dbb +size 36523 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bfe516659198de7a9fd0d8bfe1f961804d9c0e00 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36fe588a2cb429b6655f5fff6253fec5d3272fa98125ffd1f84bdf0bab0f63e5 +size 44134 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..44574a69046cb62cedeffee56c34ae91cf160c7f --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04b725bd69e0078cc6becbe04958d8492d7845a290d72ade34db03eeaebb9464 +size 52784 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ccebe81457b6d7a75583ad0bd3e1a080d3be65b --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2febc81d0fede4f6fe42ca4232395798b183358193e1d63a71264b1f57a126af +size 62516 diff --git a/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e78525e313a8a97bcbc16e035b73b4e69949b1f6 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/44120379_8371960244_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0cd73a22d565e467441ed367a794237e5aeb51c64e12d31eba36e1109f009f5 +size 72003 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..508aa2e17c3eafd0cbfdbeb5f7e8383afd1eff49 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8fd1041d9aa8b9236dc9a0fe637d96972897295236961984402858a8688dea4 +size 19103 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f749213b5d42af17b5e8fa85be5f2a0099900482 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d421ef6ed9d78b08d0b4796cfc604d3511a274e2c4be4bd3e910efcc236f07 +size 34317 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0af327e6da018baaf5e862a25b002fec00ee1481 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5480d172fc730905eb0de44d779ec45ecd11bd20907310833490eb8e816674e +size 45635 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..87a0920fc65ff0f224724e182baffa5efabb2727 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c139a0636fd9bb1c78c6999e09bd5f7229359a618754244553f1f3ff225e503 +size 55257 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c9429d3921a2f46d2bc2fa1b845c7b5ca8c34b4d --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c7f19e2d7bf07bd24785cd33a4b39a81551b0af81d6478a8e6caffd9c8702a4 +size 68433 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..816eb35ce7690a8f338698a32bf882f77c226515 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a54a79a52a214e511e7677570a548d84949989279e9d45f1ef2a3b64ec706d4 +size 83428 diff --git a/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..37c041c8c84ee938aa69efdba406855e71853961 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/51091044_3486849416_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b0fccc48763c01af6114d224630f0e9e9efb3545a4a44c0ea5d0bfc9388bc6 +size 97890 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a2fac4209480c24b542cdfae90539ca30cb34730 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0621b195be135487a03013a9bbefd1c8d06219e89255cff3e9a7f353ddf512 +size 20967 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b9bb9631981f5525ccab16e8c57e9a5fc1cd8e23 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f463bf0ca416815588402256bc118649a4021a3613fda9fee7a498fe3c088211 +size 36897 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ef911222845da2a6392efd777b08888b386e36ff --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98f2e11e0dfdc72f62b23da6b67f233a57625c9bf1b3c71d4cb832749754cc8b +size 46863 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b3ed4111f137eba6204f7678b434da1a38b8b5f --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa19ae2f619f40298b7d5afe1dd9828a870f48fd288d63d5ea348b2a86da89c +size 58545 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd502a98dd92d22c8246e9a2febad4e63fd49573 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44a1ff7cfbd900bbac8ad369232ff71b559a154e05f1f6fb43c5da24f248952d +size 70581 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e746d3e8df82d3022cb1e3192f51b067f8d22020 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:351b645bd3243f8676059d5096249bde326df7794c43c0910dc63c704d77dc28 +size 83757 diff --git a/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cf18599607edcacf5f706fdb677fe63d2c6d2b37 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/60584745_2207571072_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:592c76689f52ed651ee40640a85e826597c6a24b548b67183cc0cb556b4727e4 +size 97236 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cf7917d922cdbd30d8a415c3f71c4f1970aaace5 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5240431398627407ddf7f59f7d213ccf910ddf749bfe1d54f421e917b072e53b +size 17605 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7f0604df7dde207aa3571ad68eeb37ea9c9646f6 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3436f60a6f7f69affcd67ac82c29845f31637420be8948baa938907c7e5f0a7 +size 28290 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fee6de7cbea4e841d6811d89a80baf7c8592b79c --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb8aa81f49cda021a0d92c97ff05f960274ab442f652d6041b163f86e2b35778 +size 37127 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b6fcf0e854278c71762b85c0eff8b74a72a8cf78 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8e931c5c4dfe453042cca041f37fa9cdc168fbe83f7a2ccd3f7a8aced93a44e +size 45866 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d2b263a84eb2fa2ecc1ff8e8e516ee47d8676904 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60fc4db91f0f34837601f066cd334840ce391b6d78b524ce28b7cc6a328227e2 +size 53956 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7ec774e14c9d0a09ba1d51f8834f5719a132b89e --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff223aa66f53e55ecd076c15ccb9395bee4d1baa8286d2acf236979787417626 +size 63961 diff --git a/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7c9d39b86fe5e72e9fe4061af79fe9bda7096a86 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/71295362_4051449754_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c54568031f5558c7dfff0bb2ed2dcc679a97f89045512f44cb24869d7a2783d5 +size 74563 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.2.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..554e7360ffea8643309bc60a4397b6f0a33600b1 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d5e5f2d3cf4b6ebb1a33cf667622d5f10f4a6f6905b6d317c8793368768747b +size 18058 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.4.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b9c001354e3160e3849485eb79cfd9bd91bd559b --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7613d4ae7a19dd03b69cce7ae7d9084568b96de51a6008a0e7ba1684b2e1d18 +size 28921 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.5.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..654b114c83a846ddc7d3004d4ecb33bb48f9dd95 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120f479a4a45f31a7d01a4401c7fa96a2028a2521049bd4c9effd141ac3c5498 +size 36174 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.6.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..548a5f7d5ed22f5a93fd58723d47a979e3d0a9fe --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d490f0baec9460ad46f4fbfab2fe8fa8f8db37019e1c055300008f3de9421ec +size 43542 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.7.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a6ff24a52f698f00033277c0fb149c1e3ea92fea --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68697e9c9cda399c66c43efad0c89ea2500ef50b86be536cea9cdaeebd3742db +size 53236 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.8.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cad1a64516428929de5303d83eb9cd0cc6592f42 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:161e5e0aa1681fb3de437895f262811260bc7e7632245094663c8d527ed7634d +size 61754 diff --git a/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.9.jpg b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f7e9b31632c13cf41edcfe060442ec44acdae202 --- /dev/null +++ b/datasets/sacre_coeur/mapping_scale/93341989_396310999_scale0.9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf1f3fd9123cd4d9246dadefdd6b25e3ee8efddd070aa9642b79545b5845898d +size 72921 diff --git a/hloc/__init__.py b/hloc/__init__.py index 619292e3d0e8699054bc94c57f72542e51e5a980..172b949b16c98b5e3eb9765d64fabd56906b38cf 100644 --- a/hloc/__init__.py +++ b/hloc/__init__.py @@ -1,4 +1,5 @@ import logging +import torch from packaging import version __version__ = "1.3" @@ -30,3 +31,5 @@ else: minimal_version, found_version, ) + +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/hloc/matchers/roma.py b/hloc/matchers/roma.py index 2c4a7b0dd78f1ba9482410eff84a1bf68fd44f3b..30964a735b372f9e38158193bc0d1748537ae8b8 100644 --- a/hloc/matchers/roma.py +++ b/hloc/matchers/roma.py @@ -8,7 +8,7 @@ from .. import logger roma_path = Path(__file__).parent / "../../third_party/RoMa" sys.path.append(str(roma_path)) -from roma.models.model_zoo import roma_model +from romatch.models.model_zoo import roma_model device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -19,6 +19,7 @@ class Roma(BaseModel): "model_name": "roma_outdoor.pth", "model_utils_name": "dinov2_vitl14_pretrain.pth", "max_keypoints": 3000, + "tiny_roma": None, } required_inputs = [ "image0", diff --git a/hloc/matchers/sgmnet.py b/hloc/matchers/sgmnet.py index 22566b42d47275ffb4b2e8306d3cac4017339cda..7931a6bd51c415a374ba44d3f83775a1a2879f9f 100644 --- a/hloc/matchers/sgmnet.py +++ b/hloc/matchers/sgmnet.py @@ -5,7 +5,6 @@ import torch from PIL import Image from collections import OrderedDict, namedtuple from ..utils.base_model import BaseModel -from ..utils import do_system from .. import logger sgmnet_path = Path(__file__).parent / "../../third_party/SGMNet" @@ -72,9 +71,9 @@ class SGMNet(BaseModel): except subprocess.CalledProcessError as e: logger.error(f"Failed to download the SGMNet model.") raise e - cmd = [f"cd {str(sgmnet_path)} & tar -xvf", str(tar_path)] + cmd = ["tar", "-xvf", str(tar_path), "-C", str(sgmnet_path)] logger.info(f"Unzip model file `{cmd}`.") - do_system(f"cd {str(sgmnet_path)} & tar -xvf {str(tar_path)}") + subprocess.run(cmd, check=True) # config config = namedtuple("config", conf.keys())(*conf.values()) diff --git a/requirements.txt b/requirements.txt index 70213dfb49b7fdbd8c849a96caec0364ea6e6c8e..28194fea5246b17c39c2f8b9bfe2738081b9be0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,4 +32,5 @@ torchvision==0.17.1 tqdm==4.65.0 yacs==0.1.8 onnxruntime -poselib \ No newline at end of file +poselib +roma #dust3r \ No newline at end of file diff --git a/third_party/RoMa/.gitignore b/third_party/RoMa/.gitignore index ff4633046059da69ab4b6e222909614ccda82ac4..7f8801c4b89dd12eeca1e5709e8161f017370a44 100644 --- a/third_party/RoMa/.gitignore +++ b/third_party/RoMa/.gitignore @@ -2,4 +2,10 @@ *.vscode* *__pycache__* vis* -workspace* \ No newline at end of file +workspace* +.venv +.DS_Store +jobs/* +*ignore_me* +*.pth +wandb* \ No newline at end of file diff --git a/third_party/RoMa/README.md b/third_party/RoMa/README.md index a3b6484b2a6c19af426b731396c5c91331f99ada..8458762622603dc8d0b4a89ba5cb3354bee7deb5 100644 --- a/third_party/RoMa/README.md +++ b/third_party/RoMa/README.md @@ -34,7 +34,7 @@ pip install -e . We provide two demos in the [demos folder](demo). Here's the gist of it: ```python -from roma import roma_outdoor +from romatch import roma_outdoor roma_model = roma_outdoor(device=device) # Match warp, certainty = roma_model.match(imA_path, imB_path, device=device) @@ -48,7 +48,8 @@ F, mask = cv2.findFundamentalMat( ) ``` -**New**: You can also match arbitrary keypoints with RoMa. A demo for this will be added soon. +**New**: You can also match arbitrary keypoints with RoMa. See [match_keypoints](romatch/models/matcher.py) in RegressionMatcher. + ## Settings ### Resolution @@ -80,6 +81,36 @@ DINOv2 has an Apache 2 license [DINOv2](https://github.com/facebookresearch/dino ## Acknowledgement Our codebase builds on the code in [DKM](https://github.com/Parskatt/DKM). +## Tiny RoMa +If you find that RoMa is too heavy, you might want to try Tiny RoMa which is built on top of XFeat. +```python +from romatch import tiny_roma_v1_outdoor +tiny_roma_model = tiny_roma_v1_outdoor(device=device) +``` +Mega1500: +| | AUC@5 | AUC@10 | AUC@20 | +|----------|----------|----------|----------| +| XFeat | 46.4 | 58.9 | 69.2 | +| XFeat* | 51.9 | 67.2 | 78.9 | +| Tiny RoMa v1 | 56.4 | 69.5 | 79.5 | +| RoMa | - | - | - | + +Mega-8-Scenes (See DKM): +| | AUC@5 | AUC@10 | AUC@20 | +|----------|----------|----------|----------| +| XFeat | - | - | - | +| XFeat* | 50.1 | 64.4 | 75.2 | +| Tiny RoMa v1 | 57.7 | 70.5 | 79.6 | +| RoMa | - | - | - | + +IMC22 :'): +| | mAA@10 | +|----------|----------| +| XFeat | 42.1 | +| XFeat* | - | +| Tiny RoMa v1 | 42.2 | +| RoMa | - | + ## BibTeX If you find our models useful, please consider citing our paper! ``` diff --git a/third_party/RoMa/demo/demo_3D_effect.py b/third_party/RoMa/demo/demo_3D_effect.py index 5afd6e5ce0fdd32788160e8c24df0b26a27f34dd..c6c6d1a5f96e79be698ddf312f48d5cba6b93f7d 100644 --- a/third_party/RoMa/demo/demo_3D_effect.py +++ b/third_party/RoMa/demo/demo_3D_effect.py @@ -2,9 +2,9 @@ from PIL import Image import torch import torch.nn.functional as F import numpy as np -from roma.utils.utils import tensor_to_pil +from romatch.utils.utils import tensor_to_pil -from roma import roma_outdoor +from romatch import roma_outdoor device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') diff --git a/third_party/RoMa/demo/demo_fundamental.py b/third_party/RoMa/demo/demo_fundamental.py index 31618d4b06cd56fdd4be9065fb00b826a19e10f9..ae5dc39f90ad851cfcd52ef7f6329fd655d01286 100644 --- a/third_party/RoMa/demo/demo_fundamental.py +++ b/third_party/RoMa/demo/demo_fundamental.py @@ -1,7 +1,7 @@ from PIL import Image import torch import cv2 -from roma import roma_outdoor +from romatch import roma_outdoor device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') diff --git a/third_party/RoMa/demo/demo_match.py b/third_party/RoMa/demo/demo_match.py index 80dfcd252e6665246a1b21cca7c8c64a183fa0e2..20509160e2b1806cea9f1b7beac3da32069800c6 100644 --- a/third_party/RoMa/demo/demo_match.py +++ b/third_party/RoMa/demo/demo_match.py @@ -2,9 +2,9 @@ from PIL import Image import torch import torch.nn.functional as F import numpy as np -from roma.utils.utils import tensor_to_pil +from romatch.utils.utils import tensor_to_pil -from roma import roma_outdoor +from romatch import roma_outdoor device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') diff --git a/third_party/RoMa/requirements.txt b/third_party/RoMa/requirements.txt index f0dbab3d4cb35a5f00e3dbc8e3f8b00a3e578428..81360b228245bc573dba770458e4dbeb76d0dd9f 100644 --- a/third_party/RoMa/requirements.txt +++ b/third_party/RoMa/requirements.txt @@ -10,4 +10,5 @@ matplotlib h5py wandb timm +poselib #xformers # Optional, used for memefficient attention \ No newline at end of file diff --git a/third_party/RoMa/roma/models/__init__.py b/third_party/RoMa/roma/models/__init__.py deleted file mode 100644 index 5f20461e2f3a1722e558cefab94c5164be8842c3..0000000000000000000000000000000000000000 --- a/third_party/RoMa/roma/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .model_zoo import roma_outdoor, roma_indoor \ No newline at end of file diff --git a/third_party/RoMa/roma/utils/kde.py b/third_party/RoMa/roma/utils/kde.py deleted file mode 100644 index 6ee1378282965ab091b77c2a97f0e80bd13d4637..0000000000000000000000000000000000000000 --- a/third_party/RoMa/roma/utils/kde.py +++ /dev/null @@ -1,8 +0,0 @@ -import torch - -def kde(x, std = 0.1): - # use a gaussian kernel to estimate density - x = x.half() # Do it in half precision TODO: remove hardcoding - scores = (-torch.cdist(x,x)**2/(2*std**2)).exp() - density = scores.sum(dim=-1) - return density \ No newline at end of file diff --git a/third_party/RoMa/roma/__init__.py b/third_party/RoMa/romatch/__init__.py similarity index 64% rename from third_party/RoMa/roma/__init__.py rename to third_party/RoMa/romatch/__init__.py index a7c96481e0a808b68c7b3054a3e34fa0b5c45ab9..45c2aff7bb510b1d8dfcdc9c4d91ffc3aa5021f6 100644 --- a/third_party/RoMa/roma/__init__.py +++ b/third_party/RoMa/romatch/__init__.py @@ -1,5 +1,5 @@ import os -from .models import roma_outdoor, roma_indoor +from .models import roma_outdoor, tiny_roma_v1_outdoor, roma_indoor DEBUG_MODE = False RANK = int(os.environ.get('RANK', default = 0)) diff --git a/third_party/RoMa/roma/benchmarks/__init__.py b/third_party/RoMa/romatch/benchmarks/__init__.py similarity index 64% rename from third_party/RoMa/roma/benchmarks/__init__.py rename to third_party/RoMa/romatch/benchmarks/__init__.py index de7b841a5a6ab2ba91297a181a79dfaa91c9e104..f6008f1d59371fff9a1d9b0321ff96abf4b9c87a 100644 --- a/third_party/RoMa/roma/benchmarks/__init__.py +++ b/third_party/RoMa/romatch/benchmarks/__init__.py @@ -2,3 +2,5 @@ from .hpatches_sequences_homog_benchmark import HpatchesHomogBenchmark from .scannet_benchmark import ScanNetBenchmark from .megadepth_pose_estimation_benchmark import MegaDepthPoseEstimationBenchmark from .megadepth_dense_benchmark import MegadepthDenseBenchmark +from .megadepth_pose_estimation_benchmark_poselib import Mega1500PoseLibBenchmark +from .scannet_benchmark_poselib import ScanNetPoselibBenchmark \ No newline at end of file diff --git a/third_party/RoMa/roma/benchmarks/hpatches_sequences_homog_benchmark.py b/third_party/RoMa/romatch/benchmarks/hpatches_sequences_homog_benchmark.py similarity index 99% rename from third_party/RoMa/roma/benchmarks/hpatches_sequences_homog_benchmark.py rename to third_party/RoMa/romatch/benchmarks/hpatches_sequences_homog_benchmark.py index 2154a471c73d9e883c3ba8ed1b90d708f4950a63..5972361f80d4f4e5cafd8fd359c87c0433a0a5a5 100644 --- a/third_party/RoMa/roma/benchmarks/hpatches_sequences_homog_benchmark.py +++ b/third_party/RoMa/romatch/benchmarks/hpatches_sequences_homog_benchmark.py @@ -4,7 +4,7 @@ import numpy as np import os from tqdm import tqdm -from roma.utils import pose_auc +from romatch.utils import pose_auc import cv2 diff --git a/third_party/RoMa/roma/benchmarks/megadepth_dense_benchmark.py b/third_party/RoMa/romatch/benchmarks/megadepth_dense_benchmark.py similarity index 93% rename from third_party/RoMa/roma/benchmarks/megadepth_dense_benchmark.py rename to third_party/RoMa/romatch/benchmarks/megadepth_dense_benchmark.py index 0600d354b1d0dfa7f8e2b0f8882a4cc08fafeed9..09d0a297f2937afc609eed3a74aa0c3c4c7ccebc 100644 --- a/third_party/RoMa/roma/benchmarks/megadepth_dense_benchmark.py +++ b/third_party/RoMa/romatch/benchmarks/megadepth_dense_benchmark.py @@ -1,10 +1,10 @@ import torch import numpy as np import tqdm -from roma.datasets import MegadepthBuilder -from roma.utils import warp_kpts +from romatch.datasets import MegadepthBuilder +from romatch.utils import warp_kpts from torch.utils.data import ConcatDataset -import roma +import romatch class MegadepthDenseBenchmark: def __init__(self, data_root="data/megadepth", h = 384, w = 512, num_samples = 2000) -> None: @@ -55,10 +55,10 @@ class MegadepthDenseBenchmark: dataloader = torch.utils.data.DataLoader( self.dataset, batch_size=B, num_workers=batch_size, sampler=sampler ) - for idx, data in tqdm.tqdm(enumerate(dataloader), disable = roma.RANK > 0): + for idx, data in tqdm.tqdm(enumerate(dataloader), disable = romatch.RANK > 0): im_A, im_B, depth1, depth2, T_1to2, K1, K2 = ( - data["im_A"], - data["im_B"], + data["im_A"].cuda(), + data["im_B"].cuda(), data["im_A_depth"].cuda(), data["im_B_depth"].cuda(), data["T_1to2"].cuda(), @@ -69,8 +69,8 @@ class MegadepthDenseBenchmark: gd, pck_1, pck_3, pck_5, prob = self.geometric_dist( depth1, depth2, T_1to2, K1, K2, matches ) - if roma.DEBUG_MODE: - from roma.utils.utils import tensor_to_pil + if romatch.DEBUG_MODE: + from romatch.utils.utils import tensor_to_pil import torch.nn.functional as F path = "vis" H, W = model.get_output_resolution() diff --git a/third_party/RoMa/roma/benchmarks/megadepth_pose_estimation_benchmark.py b/third_party/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark.py similarity index 99% rename from third_party/RoMa/roma/benchmarks/megadepth_pose_estimation_benchmark.py rename to third_party/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark.py index 217aebab4cb73471cc156de9e8d3d882a1b2af95..36f293f9556d919643f6f39156314a5b402d9082 100644 --- a/third_party/RoMa/roma/benchmarks/megadepth_pose_estimation_benchmark.py +++ b/third_party/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark.py @@ -1,10 +1,10 @@ import numpy as np import torch -from roma.utils import * +from romatch.utils import * from PIL import Image from tqdm import tqdm import torch.nn.functional as F -import roma +import romatch import kornia.geometry.epipolar as kepi class MegaDepthPoseEstimationBenchmark: diff --git a/third_party/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark_poselib.py b/third_party/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark_poselib.py new file mode 100644 index 0000000000000000000000000000000000000000..4732ccf2af5b50e6db60831d7c63c5bf70ec727c --- /dev/null +++ b/third_party/RoMa/romatch/benchmarks/megadepth_pose_estimation_benchmark_poselib.py @@ -0,0 +1,119 @@ +import numpy as np +import torch +from romatch.utils import * +from PIL import Image +from tqdm import tqdm +import torch.nn.functional as F +import romatch +import kornia.geometry.epipolar as kepi + +# wrap cause pyposelib is still in dev +# will add in deps later +import poselib + +class Mega1500PoseLibBenchmark: + def __init__(self, data_root="data/megadepth", scene_names = None, num_ransac_iter = 5, test_every = 1) -> None: + if scene_names is None: + self.scene_names = [ + "0015_0.1_0.3.npz", + "0015_0.3_0.5.npz", + "0022_0.1_0.3.npz", + "0022_0.3_0.5.npz", + "0022_0.5_0.7.npz", + ] + else: + self.scene_names = scene_names + self.scenes = [ + np.load(f"{data_root}/{scene}", allow_pickle=True) + for scene in self.scene_names + ] + self.data_root = data_root + self.num_ransac_iter = num_ransac_iter + self.test_every = test_every + + def benchmark(self, model, model_name = None): + with torch.no_grad(): + data_root = self.data_root + tot_e_t, tot_e_R, tot_e_pose = [], [], [] + thresholds = [5, 10, 20] + for scene_ind in range(len(self.scenes)): + import os + scene_name = os.path.splitext(self.scene_names[scene_ind])[0] + scene = self.scenes[scene_ind] + pairs = scene["pair_infos"] + intrinsics = scene["intrinsics"] + poses = scene["poses"] + im_paths = scene["image_paths"] + pair_inds = range(len(pairs))[::self.test_every] + for pairind in (pbar := tqdm(pair_inds, desc = "Current AUC: ?")): + idx1, idx2 = pairs[pairind][0] + K1 = intrinsics[idx1].copy() + T1 = poses[idx1].copy() + R1, t1 = T1[:3, :3], T1[:3, 3] + K2 = intrinsics[idx2].copy() + T2 = poses[idx2].copy() + R2, t2 = T2[:3, :3], T2[:3, 3] + R, t = compute_relative_pose(R1, t1, R2, t2) + T1_to_2 = np.concatenate((R,t[:,None]), axis=-1) + im_A_path = f"{data_root}/{im_paths[idx1]}" + im_B_path = f"{data_root}/{im_paths[idx2]}" + dense_matches, dense_certainty = model.match( + im_A_path, im_B_path, K1.copy(), K2.copy(), T1_to_2.copy() + ) + sparse_matches,_ = model.sample( + dense_matches, dense_certainty, 5_000 + ) + + im_A = Image.open(im_A_path) + w1, h1 = im_A.size + im_B = Image.open(im_B_path) + w2, h2 = im_B.size + kpts1, kpts2 = model.to_pixel_coordinates(sparse_matches, h1, w1, h2, w2) + kpts1, kpts2 = kpts1.cpu().numpy(), kpts2.cpu().numpy() + for _ in range(self.num_ransac_iter): + shuffling = np.random.permutation(np.arange(len(kpts1))) + kpts1 = kpts1[shuffling] + kpts2 = kpts2[shuffling] + try: + threshold = 1 + camera1 = {'model': 'PINHOLE', 'width': w1, 'height': h1, 'params': K1[[0,1,0,1], [0,1,2,2]]} + camera2 = {'model': 'PINHOLE', 'width': w2, 'height': h2, 'params': K2[[0,1,0,1], [0,1,2,2]]} + relpose, res = poselib.estimate_relative_pose( + kpts1, + kpts2, + camera1, + camera2, + ransac_opt = {"max_reproj_error": 2*threshold, "max_epipolar_error": threshold, "min_inliers": 8, "max_iterations": 10_000}, + ) + Rt_est = relpose.Rt + R_est, t_est = Rt_est[:3,:3], Rt_est[:3,3:] + mask = np.array(res['inliers']).astype(np.float32) + T1_to_2_est = np.concatenate((R_est, t_est), axis=-1) # + e_t, e_R = compute_pose_error(T1_to_2_est, R, t) + e_pose = max(e_t, e_R) + except Exception as e: + print(repr(e)) + e_t, e_R = 90, 90 + e_pose = max(e_t, e_R) + tot_e_t.append(e_t) + tot_e_R.append(e_R) + tot_e_pose.append(e_pose) + pbar.set_description(f"Current AUC: {pose_auc(tot_e_pose, thresholds)}") + tot_e_pose = np.array(tot_e_pose) + auc = pose_auc(tot_e_pose, thresholds) + acc_5 = (tot_e_pose < 5).mean() + acc_10 = (tot_e_pose < 10).mean() + acc_15 = (tot_e_pose < 15).mean() + acc_20 = (tot_e_pose < 20).mean() + map_5 = acc_5 + map_10 = np.mean([acc_5, acc_10]) + map_20 = np.mean([acc_5, acc_10, acc_15, acc_20]) + print(f"{model_name} auc: {auc}") + return { + "auc_5": auc[0], + "auc_10": auc[1], + "auc_20": auc[2], + "map_5": map_5, + "map_10": map_10, + "map_20": map_20, + } diff --git a/third_party/RoMa/roma/benchmarks/scannet_benchmark.py b/third_party/RoMa/romatch/benchmarks/scannet_benchmark.py similarity index 99% rename from third_party/RoMa/roma/benchmarks/scannet_benchmark.py rename to third_party/RoMa/romatch/benchmarks/scannet_benchmark.py index 853af0d0ebef4dfefe2632eb49e4156ea791ee76..8c6b3c0eeb1c211d224edde84974529c66b52460 100644 --- a/third_party/RoMa/roma/benchmarks/scannet_benchmark.py +++ b/third_party/RoMa/romatch/benchmarks/scannet_benchmark.py @@ -1,7 +1,7 @@ import os.path as osp import numpy as np import torch -from roma.utils import * +from romatch.utils import * from PIL import Image from tqdm import tqdm diff --git a/third_party/RoMa/roma/checkpointing/__init__.py b/third_party/RoMa/romatch/checkpointing/__init__.py similarity index 100% rename from third_party/RoMa/roma/checkpointing/__init__.py rename to third_party/RoMa/romatch/checkpointing/__init__.py diff --git a/third_party/RoMa/roma/checkpointing/checkpoint.py b/third_party/RoMa/romatch/checkpointing/checkpoint.py similarity index 96% rename from third_party/RoMa/roma/checkpointing/checkpoint.py rename to third_party/RoMa/romatch/checkpointing/checkpoint.py index 8995efeb54f4d558127ea63423fa958c64e9088f..ab5a5131322241475323f1b992d9d3f7b21dbdac 100644 --- a/third_party/RoMa/roma/checkpointing/checkpoint.py +++ b/third_party/RoMa/romatch/checkpointing/checkpoint.py @@ -5,7 +5,7 @@ from torch.nn.parallel.distributed import DistributedDataParallel from loguru import logger import gc -import roma +import romatch class CheckPoint: def __init__(self, dir=None, name="tmp"): @@ -20,7 +20,7 @@ class CheckPoint: lr_scheduler, n, ): - if roma.RANK == 0: + if romatch.RANK == 0: assert model is not None if isinstance(model, (DataParallel, DistributedDataParallel)): model = model.module @@ -40,7 +40,7 @@ class CheckPoint: lr_scheduler, n, ): - if os.path.exists(self.dir + self.name + f"_latest.pth") and roma.RANK == 0: + if os.path.exists(self.dir + self.name + f"_latest.pth") and romatch.RANK == 0: states = torch.load(self.dir + self.name + f"_latest.pth") if "model" in states: model.load_state_dict(states["model"]) diff --git a/third_party/RoMa/roma/datasets/__init__.py b/third_party/RoMa/romatch/datasets/__init__.py similarity index 100% rename from third_party/RoMa/roma/datasets/__init__.py rename to third_party/RoMa/romatch/datasets/__init__.py diff --git a/third_party/RoMa/roma/datasets/megadepth.py b/third_party/RoMa/romatch/datasets/megadepth.py similarity index 97% rename from third_party/RoMa/roma/datasets/megadepth.py rename to third_party/RoMa/romatch/datasets/megadepth.py index 5deee5ac30c439a9f300c0ad2271f141931020c0..88f775ef412f7bd062cc9a1d67d95a030e7a15dd 100644 --- a/third_party/RoMa/roma/datasets/megadepth.py +++ b/third_party/RoMa/romatch/datasets/megadepth.py @@ -5,9 +5,9 @@ import numpy as np import torch import torchvision.transforms.functional as tvf import kornia.augmentation as K -from roma.utils import get_depth_tuple_transform_ops, get_tuple_transform_ops -import roma -from roma.utils import * +from romatch.utils import get_depth_tuple_transform_ops, get_tuple_transform_ops +import romatch +from romatch.utils import * import math class MegadepthScene: @@ -53,7 +53,7 @@ class MegadepthScene: area = ht * wt s = int(16 * (math.sqrt(area)//16)) sizes = ((ht,wt), (s,s), (wt,ht)) - choice = roma.RANK % 3 + choice = romatch.RANK % 3 ht, wt = sizes[choice] # counts, bins = np.histogram(self.overlaps,20) # print(counts) @@ -157,7 +157,7 @@ class MegadepthScene: if np.random.rand() > 0.5: im_B, depth_B, K2 = self.single_horizontal_flip(im_B, depth_B, K2) - if roma.DEBUG_MODE: + if romatch.DEBUG_MODE: tensor_to_pil(im_A[0], unnormalize=True).save( f"vis/im_A.jpg") tensor_to_pil(im_B[0], unnormalize=True).save( @@ -212,6 +212,8 @@ class MegadepthBuilder: continue if self.imc21_ignore and scene_name in self.imc21_scenes: continue + if ".npy" not in scene_name: + continue scene_info = np.load( os.path.join(self.scene_info_root, scene_name), allow_pickle=True ).item() diff --git a/third_party/RoMa/roma/datasets/scannet.py b/third_party/RoMa/romatch/datasets/scannet.py similarity index 96% rename from third_party/RoMa/roma/datasets/scannet.py rename to third_party/RoMa/romatch/datasets/scannet.py index 704ea57259afdfbbca627ad143bee97a0a79d41c..e03261557147cd3449c76576a5e5e22c0ae288e9 100644 --- a/third_party/RoMa/roma/datasets/scannet.py +++ b/third_party/RoMa/romatch/datasets/scannet.py @@ -14,9 +14,9 @@ import torchvision.transforms.functional as tvf import kornia.augmentation as K import os.path as osp import matplotlib.pyplot as plt -import roma -from roma.utils import get_depth_tuple_transform_ops, get_tuple_transform_ops -from roma.utils.transforms import GeometricSequential +import romatch +from romatch.utils import get_depth_tuple_transform_ops, get_tuple_transform_ops +from romatch.utils.transforms import GeometricSequential from tqdm import tqdm class ScanNetScene: @@ -147,7 +147,7 @@ class ScanNetBuilder: # Note: split doesn't matter here as we always use same scannet_train scenes scene_names = self.all_scenes scenes = [] - for scene_name in tqdm(scene_names, disable = roma.RANK > 0): + for scene_name in tqdm(scene_names, disable = romatch.RANK > 0): scene_info = np.load(os.path.join(self.scene_info_root,scene_name), allow_pickle=True) scenes.append(ScanNetScene(self.data_root, scene_info, min_overlap=min_overlap, **kwargs)) return scenes diff --git a/third_party/RoMa/roma/losses/__init__.py b/third_party/RoMa/romatch/losses/__init__.py similarity index 100% rename from third_party/RoMa/roma/losses/__init__.py rename to third_party/RoMa/romatch/losses/__init__.py diff --git a/third_party/RoMa/roma/losses/robust_loss.py b/third_party/RoMa/romatch/losses/robust_loss.py similarity index 90% rename from third_party/RoMa/roma/losses/robust_loss.py rename to third_party/RoMa/romatch/losses/robust_loss.py index b932b2706f619c083485e1be0d86eec44ead83ef..4a9988c02b689c66caf2c0262a470e216c62857f 100644 --- a/third_party/RoMa/roma/losses/robust_loss.py +++ b/third_party/RoMa/romatch/losses/robust_loss.py @@ -2,9 +2,9 @@ from einops.einops import rearrange import torch import torch.nn as nn import torch.nn.functional as F -from roma.utils.utils import get_gt_warp +from romatch.utils.utils import get_gt_warp import wandb -import roma +import romatch import math class RobustLosses(nn.Module): @@ -49,15 +49,15 @@ class RobustLosses(nn.Module): G = torch.stack((G[1], G[0]), dim = -1).reshape(C,2) GT = (G[None,:,None,None,:]-x2[:,None]).norm(dim=-1).min(dim=1).indices cls_loss = F.cross_entropy(scale_gm_cls, GT, reduction = 'none')[prob > 0.99] + certainty_loss = F.binary_cross_entropy_with_logits(gm_certainty[:,0], prob) if not torch.any(cls_loss): cls_loss = (certainty_loss * 0.0) # Prevent issues where prob is 0 everywhere - - certainty_loss = F.binary_cross_entropy_with_logits(gm_certainty[:,0], prob) + losses = { f"gm_certainty_loss_{scale}": certainty_loss.mean(), f"gm_cls_loss_{scale}": cls_loss.mean(), } - wandb.log(losses, step = roma.GLOBAL_STEP) + wandb.log(losses, step = romatch.GLOBAL_STEP) return losses def delta_cls_loss(self, x2, prob, flow_pre_delta, delta_cls, certainty, scale, offset_scale): @@ -76,17 +76,17 @@ class RobustLosses(nn.Module): f"delta_certainty_loss_{scale}": certainty_loss.mean(), f"delta_cls_loss_{scale}": cls_loss.mean(), } - wandb.log(losses, step = roma.GLOBAL_STEP) + wandb.log(losses, step = romatch.GLOBAL_STEP) return losses def regression_loss(self, x2, prob, flow, certainty, scale, eps=1e-8, mode = "delta"): epe = (flow.permute(0,2,3,1) - x2).norm(dim=-1) if scale == 1: pck_05 = (epe[prob > 0.99] < 0.5 * (2/512)).float().mean() - wandb.log({"train_pck_05": pck_05}, step = roma.GLOBAL_STEP) + wandb.log({"train_pck_05": pck_05}, step = romatch.GLOBAL_STEP) ce_loss = F.binary_cross_entropy_with_logits(certainty[:, 0], prob) - a = self.alpha + a = self.alpha[scale] if isinstance(self.alpha, dict) else self.alpha cs = self.c * scale x = epe[prob > 0.99] reg_loss = cs**a * ((x/(cs))**2 + 1**2)**(a/2) @@ -96,7 +96,7 @@ class RobustLosses(nn.Module): f"{mode}_certainty_loss_{scale}": ce_loss.mean(), f"{mode}_regression_loss_{scale}": reg_loss.mean(), } - wandb.log(losses, step = roma.GLOBAL_STEP) + wandb.log(losses, step = romatch.GLOBAL_STEP) return losses def forward(self, corresps, batch): @@ -108,7 +108,7 @@ class RobustLosses(nn.Module): scale_corresps = corresps[scale] scale_certainty, flow_pre_delta, delta_cls, offset_scale, scale_gm_cls, scale_gm_certainty, flow, scale_gm_flow = ( scale_corresps["certainty"], - scale_corresps["flow_pre_delta"], + scale_corresps.get("flow_pre_delta"), scale_corresps.get("delta_cls"), scale_corresps.get("offset_scale"), scale_corresps.get("gm_cls"), @@ -117,8 +117,12 @@ class RobustLosses(nn.Module): scale_corresps.get("gm_flow"), ) - flow_pre_delta = rearrange(flow_pre_delta, "b d h w -> b h w d") - b, h, w, d = flow_pre_delta.shape + if flow_pre_delta is not None: + flow_pre_delta = rearrange(flow_pre_delta, "b d h w -> b h w d") + b, h, w, d = flow_pre_delta.shape + else: + # _ = 1 + b, _, h, w = scale_certainty.shape gt_warp, gt_prob = get_gt_warp( batch["im_A_depth"], batch["im_B_depth"], diff --git a/third_party/RoMa/romatch/losses/robust_loss_tiny_roma.py b/third_party/RoMa/romatch/losses/robust_loss_tiny_roma.py new file mode 100644 index 0000000000000000000000000000000000000000..a17c24678b093ca843d16c1a17ea16f19fa594d5 --- /dev/null +++ b/third_party/RoMa/romatch/losses/robust_loss_tiny_roma.py @@ -0,0 +1,160 @@ +from einops.einops import rearrange +import torch +import torch.nn as nn +import torch.nn.functional as F +from romatch.utils.utils import get_gt_warp +import wandb +import romatch +import math + +# This is slightly different than regular romatch due to significantly worse corresps +# The confidence loss is quite tricky here //Johan + +class RobustLosses(nn.Module): + def __init__( + self, + robust=False, + center_coords=False, + scale_normalize=False, + ce_weight=0.01, + local_loss=True, + local_dist=None, + smooth_mask = False, + depth_interpolation_mode = "bilinear", + mask_depth_loss = False, + relative_depth_error_threshold = 0.05, + alpha = 1., + c = 1e-3, + epe_mask_prob_th = None, + cert_only_on_consistent_depth = False, + ): + super().__init__() + if local_dist is None: + local_dist = {} + self.robust = robust # measured in pixels + self.center_coords = center_coords + self.scale_normalize = scale_normalize + self.ce_weight = ce_weight + self.local_loss = local_loss + self.local_dist = local_dist + self.smooth_mask = smooth_mask + self.depth_interpolation_mode = depth_interpolation_mode + self.mask_depth_loss = mask_depth_loss + self.relative_depth_error_threshold = relative_depth_error_threshold + self.avg_overlap = dict() + self.alpha = alpha + self.c = c + self.epe_mask_prob_th = epe_mask_prob_th + self.cert_only_on_consistent_depth = cert_only_on_consistent_depth + + def corr_volume_loss(self, mnn:torch.Tensor, corr_volume:torch.Tensor, scale): + b, h,w, h,w = corr_volume.shape + inv_temp = 10 + corr_volume = corr_volume.reshape(-1, h*w, h*w) + nll = -(inv_temp*corr_volume).log_softmax(dim = 1) - (inv_temp*corr_volume).log_softmax(dim = 2) + corr_volume_loss = nll[mnn[:,0], mnn[:,1], mnn[:,2]].mean() + + losses = { + f"gm_corr_volume_loss_{scale}": corr_volume_loss.mean(), + } + wandb.log(losses, step = romatch.GLOBAL_STEP) + return losses + + + + def regression_loss(self, x2, prob, flow, certainty, scale, eps=1e-8, mode = "delta"): + epe = (flow.permute(0,2,3,1) - x2).norm(dim=-1) + if scale in self.local_dist: + prob = prob * (epe < (2 / 512) * (self.local_dist[scale] * scale)).float() + if scale == 1: + pck_05 = (epe[prob > 0.99] < 0.5 * (2/512)).float().mean() + wandb.log({"train_pck_05": pck_05}, step = romatch.GLOBAL_STEP) + if self.epe_mask_prob_th is not None: + # if too far away from gt, certainty should be 0 + gt_cert = prob * (epe < scale * self.epe_mask_prob_th) + else: + gt_cert = prob + if self.cert_only_on_consistent_depth: + ce_loss = F.binary_cross_entropy_with_logits(certainty[:, 0][prob > 0], gt_cert[prob > 0]) + else: + ce_loss = F.binary_cross_entropy_with_logits(certainty[:, 0], gt_cert) + a = self.alpha[scale] if isinstance(self.alpha, dict) else self.alpha + cs = self.c * scale + x = epe[prob > 0.99] + reg_loss = cs**a * ((x/(cs))**2 + 1**2)**(a/2) + if not torch.any(reg_loss): + reg_loss = (ce_loss * 0.0) # Prevent issues where prob is 0 everywhere + losses = { + f"{mode}_certainty_loss_{scale}": ce_loss.mean(), + f"{mode}_regression_loss_{scale}": reg_loss.mean(), + } + wandb.log(losses, step = romatch.GLOBAL_STEP) + return losses + + def forward(self, corresps, batch): + scales = list(corresps.keys()) + tot_loss = 0.0 + # scale_weights due to differences in scale for regression gradients and classification gradients + for scale in scales: + scale_corresps = corresps[scale] + scale_certainty, flow_pre_delta, delta_cls, offset_scale, scale_gm_corr_volume, scale_gm_certainty, flow, scale_gm_flow = ( + scale_corresps["certainty"], + scale_corresps.get("flow_pre_delta"), + scale_corresps.get("delta_cls"), + scale_corresps.get("offset_scale"), + scale_corresps.get("corr_volume"), + scale_corresps.get("gm_certainty"), + scale_corresps["flow"], + scale_corresps.get("gm_flow"), + + ) + if flow_pre_delta is not None: + flow_pre_delta = rearrange(flow_pre_delta, "b d h w -> b h w d") + b, h, w, d = flow_pre_delta.shape + else: + # _ = 1 + b, _, h, w = scale_certainty.shape + gt_warp, gt_prob = get_gt_warp( + batch["im_A_depth"], + batch["im_B_depth"], + batch["T_1to2"], + batch["K1"], + batch["K2"], + H=h, + W=w, + ) + x2 = gt_warp.float() + prob = gt_prob + + if scale_gm_corr_volume is not None: + gt_warp_back, _ = get_gt_warp( + batch["im_B_depth"], + batch["im_A_depth"], + batch["T_1to2"].inverse(), + batch["K2"], + batch["K1"], + H=h, + W=w, + ) + grid = torch.stack(torch.meshgrid(torch.linspace(-1+1/w, 1-1/w, w), torch.linspace(-1+1/h, 1-1/h, h), indexing='xy'), dim =-1).to(gt_warp.device) + #fwd_bck = F.grid_sample(gt_warp_back.permute(0,3,1,2), gt_warp, align_corners=False, mode = 'bilinear').permute(0,2,3,1) + #diff = (fwd_bck - grid).norm(dim = -1) + with torch.no_grad(): + D_B = torch.cdist(gt_warp.float().reshape(-1,h*w,2), grid.reshape(-1,h*w,2)) + D_A = torch.cdist(grid.reshape(-1,h*w,2), gt_warp_back.float().reshape(-1,h*w,2)) + inds = torch.nonzero((D_B == D_B.min(dim=-1, keepdim = True).values) + * (D_A == D_A.min(dim=-2, keepdim = True).values) + * (D_B < 0.01) + * (D_A < 0.01)) + + gm_cls_losses = self.corr_volume_loss(inds, scale_gm_corr_volume, scale) + gm_loss = gm_cls_losses[f"gm_corr_volume_loss_{scale}"] + tot_loss = tot_loss + gm_loss + elif scale_gm_flow is not None: + gm_flow_losses = self.regression_loss(x2, prob, scale_gm_flow, scale_gm_certainty, scale, mode = "gm") + gm_loss = self.ce_weight * gm_flow_losses[f"gm_certainty_loss_{scale}"] + gm_flow_losses[f"gm_regression_loss_{scale}"] + tot_loss = tot_loss + gm_loss + delta_regression_losses = self.regression_loss(x2, prob, flow, scale_certainty, scale) + reg_loss = self.ce_weight * delta_regression_losses[f"delta_certainty_loss_{scale}"] + delta_regression_losses[f"delta_regression_loss_{scale}"] + tot_loss = tot_loss + reg_loss + return tot_loss diff --git a/third_party/RoMa/romatch/models/__init__.py b/third_party/RoMa/romatch/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7650c9c7480920905e27578f175fcb5f995cc8ba --- /dev/null +++ b/third_party/RoMa/romatch/models/__init__.py @@ -0,0 +1 @@ +from .model_zoo import roma_outdoor, tiny_roma_v1_outdoor, roma_indoor \ No newline at end of file diff --git a/third_party/RoMa/roma/models/encoders.py b/third_party/RoMa/romatch/models/encoders.py similarity index 100% rename from third_party/RoMa/roma/models/encoders.py rename to third_party/RoMa/romatch/models/encoders.py diff --git a/third_party/RoMa/roma/models/matcher.py b/third_party/RoMa/romatch/models/matcher.py similarity index 90% rename from third_party/RoMa/roma/models/matcher.py rename to third_party/RoMa/romatch/models/matcher.py index 25a89c8dd99bc1eca8c591dbbc3b5ddbd987829c..823fc19bc79a693f9120a8dbe3162e88f0c69dc0 100644 --- a/third_party/RoMa/roma/models/matcher.py +++ b/third_party/RoMa/romatch/models/matcher.py @@ -9,11 +9,12 @@ import warnings from warnings import warn from PIL import Image -import roma -from roma.utils import get_tuple_transform_ops -from roma.utils.local_correlation import local_correlation -from roma.utils.utils import cls_to_flow_refine -from roma.utils.kde import kde +import romatch +from romatch.utils import get_tuple_transform_ops +from romatch.utils.local_correlation import local_correlation +from romatch.utils.utils import cls_to_flow_refine +from romatch.utils.kde import kde +from typing import Union class ConvRefiner(nn.Module): def __init__( @@ -524,15 +525,43 @@ class RegressionMatcher(nn.Module): scale_factor=scale_factor) return corresps - def to_pixel_coordinates(self, coords, H_A, W_A, H_B, W_B): + def conf_from_fb_consistency(self, flow_forward, flow_backward, th = 2): + # assumes that flow forward is of shape (..., H, W, 2) + has_batch = False + if len(flow_forward.shape) == 3: + flow_forward, flow_backward = flow_forward[None], flow_backward[None] + else: + has_batch = True + H,W = flow_forward.shape[-3:-1] + th_n = 2 * th / max(H,W) + coords = torch.stack(torch.meshgrid( + torch.linspace(-1 + 1 / W, 1 - 1 / W, W), + torch.linspace(-1 + 1 / H, 1 - 1 / H, H), indexing = "xy"), + dim = -1).to(flow_forward.device) + coords_fb = F.grid_sample( + flow_backward.permute(0, 3, 1, 2), + flow_forward, + align_corners=False, mode="bilinear").permute(0, 2, 3, 1) + diff = (coords - coords_fb).norm(dim=-1) + in_th = (diff < th_n).float() + if not has_batch: + in_th = in_th[0] + return in_th + + def to_pixel_coordinates(self, coords, H_A, W_A, H_B = None, W_B = None): + if coords.shape[-1] == 2: + return self._to_pixel_coordinates(coords, H_A, W_A) + if isinstance(coords, (list, tuple)): kpts_A, kpts_B = coords[0], coords[1] else: kpts_A, kpts_B = coords[...,:2], coords[...,2:] - kpts_A = torch.stack((W_A/2 * (kpts_A[...,0]+1), H_A/2 * (kpts_A[...,1]+1)),axis=-1) - kpts_B = torch.stack((W_B/2 * (kpts_B[...,0]+1), H_B/2 * (kpts_B[...,1]+1)),axis=-1) - return kpts_A, kpts_B - + return self._to_pixel_coordinates(kpts_A, H_A, W_A), self._to_pixel_coordinates(kpts_B, H_B, W_B) + + def _to_pixel_coordinates(self, coords, H, W): + kpts = torch.stack((W/2 * (coords[...,0]+1), H/2 * (coords[...,1]+1)),axis=-1) + return kpts + def to_normalized_coordinates(self, coords, H_A, W_A, H_B, W_B): if isinstance(coords, (list, tuple)): kpts_A, kpts_B = coords[0], coords[1] @@ -582,8 +611,8 @@ class RegressionMatcher(nn.Module): @torch.inference_mode() def match( self, - im_A_path, - im_B_path, + im_A_path: Union[str, os.PathLike, Image.Image], + im_B_path: Union[str, os.PathLike, Image.Image], *args, batched=False, device = None, @@ -593,8 +622,8 @@ class RegressionMatcher(nn.Module): if isinstance(im_A_path, (str, os.PathLike)): im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB") else: - # Assume its not a path - im_A, im_B = im_A_path, im_B_path + im_A, im_B = im_A_path, im_B_path + symmetric = self.symmetric self.train(False) with torch.no_grad(): @@ -644,13 +673,12 @@ class RegressionMatcher(nn.Module): resize=(hs, ws), normalize=True ) if self.recrop_upsample: + raise NotImplementedError("recrop_upsample not implemented") certainty = corresps[finest_scale]["certainty"] print(certainty.shape) im_A = self.recrop(certainty[0,0], im_A_path) im_B = self.recrop(certainty[1,0], im_B_path) #TODO: need to adjust corresps when doing this - else: - im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB") im_A, im_B = test_transform((im_A, im_B)) im_A, im_B = im_A[None].to(device), im_B[None].to(device) scale_factor = math.sqrt(self.upsample_res[0] * self.upsample_res[1] / (self.w_resized * self.h_resized)) @@ -707,29 +735,38 @@ class RegressionMatcher(nn.Module): certainty[0, 0], ) - def visualize_warp(self, warp, certainty, im_A = None, im_B = None, im_A_path = None, im_B_path = None, device = "cuda", symmetric = True, save_path = None): - assert symmetric == True, "Currently assuming bidirectional warp, might update this if someone complains ;)" + def visualize_warp(self, warp, certainty, im_A = None, im_B = None, + im_A_path = None, im_B_path = None, device = "cuda", symmetric = True, save_path = None, unnormalize = False): + #assert symmetric == True, "Currently assuming bidirectional warp, might update this if someone complains ;)" H,W2,_ = warp.shape W = W2//2 if symmetric else W2 if im_A is None: from PIL import Image im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB") - im_A = im_A.resize((W,H)) - im_B = im_B.resize((W,H)) - - x_A = (torch.tensor(np.array(im_A)) / 255).to(device).permute(2, 0, 1) - x_B = (torch.tensor(np.array(im_B)) / 255).to(device).permute(2, 0, 1) - + if not isinstance(im_A, torch.Tensor): + im_A = im_A.resize((W,H)) + im_B = im_B.resize((W,H)) + x_B = (torch.tensor(np.array(im_B)) / 255).to(device).permute(2, 0, 1) + if symmetric: + x_A = (torch.tensor(np.array(im_A)) / 255).to(device).permute(2, 0, 1) + else: + if symmetric: + x_A = im_A + x_B = im_B im_A_transfer_rgb = F.grid_sample( x_B[None], warp[:,:W, 2:][None], mode="bilinear", align_corners=False )[0] - im_B_transfer_rgb = F.grid_sample( - x_A[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False - )[0] - warp_im = torch.cat((im_A_transfer_rgb,im_B_transfer_rgb),dim=2) - white_im = torch.ones((H,2*W),device=device) + if symmetric: + im_B_transfer_rgb = F.grid_sample( + x_A[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False + )[0] + warp_im = torch.cat((im_A_transfer_rgb,im_B_transfer_rgb),dim=2) + white_im = torch.ones((H,2*W),device=device) + else: + warp_im = im_A_transfer_rgb + white_im = torch.ones((H, W), device = device) vis_im = certainty * warp_im + (1 - certainty) * white_im if save_path is not None: - from roma.utils import tensor_to_pil - tensor_to_pil(vis_im, unnormalize=False).save(save_path) + from romatch.utils import tensor_to_pil + tensor_to_pil(vis_im, unnormalize=unnormalize).save(save_path) return vis_im diff --git a/third_party/RoMa/roma/models/model_zoo/__init__.py b/third_party/RoMa/romatch/models/model_zoo/__init__.py similarity index 76% rename from third_party/RoMa/roma/models/model_zoo/__init__.py rename to third_party/RoMa/romatch/models/model_zoo/__init__.py index 49ca7b8557cb8f6948bca28c631e39d899e49177..1eacafb0dc3e76be4e6f3c75d06282f22db6f9a7 100644 --- a/third_party/RoMa/roma/models/model_zoo/__init__.py +++ b/third_party/RoMa/romatch/models/model_zoo/__init__.py @@ -1,15 +1,32 @@ from typing import Union import torch -from .roma_models import roma_model +from .roma_models import roma_model, tiny_roma_v1_model weight_urls = { - "roma": { - "outdoor": "https://github.com/Parskatt/storage/releases/download/roma/roma_outdoor.pth", - "indoor": "https://github.com/Parskatt/storage/releases/download/roma/roma_indoor.pth", + "romatch": { + "outdoor": "https://github.com/Parskatt/storage/releases/download/romatch/roma_outdoor.pth", + "indoor": "https://github.com/Parskatt/storage/releases/download/romatch/roma_indoor.pth", + }, + "tiny_roma_v1": { + "outdoor": "https://github.com/Parskatt/storage/releases/download/romatch/tiny_roma_v1_outdoor.pth", }, "dinov2": "https://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pth", #hopefully this doesnt change :D } +def tiny_roma_v1_outdoor(device, weights = None, xfeat = None): + if weights is None: + weights = torch.hub.load_state_dict_from_url( + weight_urls["tiny_roma_v1"]["outdoor"], + map_location=device) + if xfeat is None: + xfeat = torch.hub.load( + 'verlab/accelerated_features', + 'XFeat', + pretrained = True, + top_k = 4096).net + + return tiny_roma_v1_model(weights = weights, xfeat = xfeat).to(device) + def roma_outdoor(device, weights=None, dinov2_weights=None, coarse_res: Union[int,tuple[int,int]] = 560, upsample_res: Union[int,tuple[int,int]] = 864, amp_dtype: torch.dtype = torch.float16): if isinstance(coarse_res, int): coarse_res = (coarse_res, coarse_res) @@ -20,7 +37,7 @@ def roma_outdoor(device, weights=None, dinov2_weights=None, coarse_res: Union[in assert coarse_res[1] % 14 == 0, "Needs to be multiple of 14 for backbone" if weights is None: - weights = torch.hub.load_state_dict_from_url(weight_urls["roma"]["outdoor"], + weights = torch.hub.load_state_dict_from_url(weight_urls["romatch"]["outdoor"], map_location=device) if dinov2_weights is None: dinov2_weights = torch.hub.load_state_dict_from_url(weight_urls["dinov2"], @@ -41,7 +58,7 @@ def roma_indoor(device, weights=None, dinov2_weights=None, coarse_res: Union[int assert coarse_res[1] % 14 == 0, "Needs to be multiple of 14 for backbone" if weights is None: - weights = torch.hub.load_state_dict_from_url(weight_urls["roma"]["indoor"], + weights = torch.hub.load_state_dict_from_url(weight_urls["romatch"]["indoor"], map_location=device) if dinov2_weights is None: dinov2_weights = torch.hub.load_state_dict_from_url(weight_urls["dinov2"], diff --git a/third_party/RoMa/roma/models/model_zoo/roma_models.py b/third_party/RoMa/romatch/models/model_zoo/roma_models.py similarity index 90% rename from third_party/RoMa/roma/models/model_zoo/roma_models.py rename to third_party/RoMa/romatch/models/model_zoo/roma_models.py index 13f8d872f3aad6ef42b090b123f77a96ff1ce68f..4f8a08fc26d760a09f048bdd98bf8a8fffc8202c 100644 --- a/third_party/RoMa/roma/models/model_zoo/roma_models.py +++ b/third_party/RoMa/romatch/models/model_zoo/roma_models.py @@ -1,12 +1,22 @@ import warnings import torch.nn as nn import torch -from roma.models.matcher import * -from roma.models.transformer import Block, TransformerDecoder, MemEffAttention -from roma.models.encoders import * +from romatch.models.matcher import * +from romatch.models.transformer import Block, TransformerDecoder, MemEffAttention +from romatch.models.encoders import * +from romatch.models.tiny import TinyRoMa + +def tiny_roma_v1_model(weights = None, freeze_xfeat=False, exact_softmax=False, xfeat = None): + model = TinyRoMa( + xfeat = xfeat, + freeze_xfeat=freeze_xfeat, + exact_softmax=exact_softmax) + if weights is not None: + model.load_state_dict(weights) + return model def roma_model(resolution, upsample_preds, device = None, weights=None, dinov2_weights=None, amp_dtype: torch.dtype=torch.float16, **kwargs): - # roma weights and dinov2 weights are loaded seperately, as dinov2 weights are not parameters + # romatch weights and dinov2 weights are loaded seperately, as dinov2 weights are not parameters #torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul TODO: these probably ruin stuff, should be careful #torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated') diff --git a/third_party/RoMa/romatch/models/tiny.py b/third_party/RoMa/romatch/models/tiny.py new file mode 100644 index 0000000000000000000000000000000000000000..88f44af6c9a6255831734d096167724f89d040ce --- /dev/null +++ b/third_party/RoMa/romatch/models/tiny.py @@ -0,0 +1,304 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import os +import torch +from pathlib import Path +import math +import numpy as np + +from torch import nn +from PIL import Image +from torchvision.transforms import ToTensor +from romatch.utils.kde import kde + +class BasicLayer(nn.Module): + """ + Basic Convolutional Layer: Conv2d -> BatchNorm -> ReLU + """ + def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1, dilation=1, bias=False, relu = True): + super().__init__() + self.layer = nn.Sequential( + nn.Conv2d( in_channels, out_channels, kernel_size, padding = padding, stride=stride, dilation=dilation, bias = bias), + nn.BatchNorm2d(out_channels, affine=False), + nn.ReLU(inplace = True) if relu else nn.Identity() + ) + + def forward(self, x): + return self.layer(x) + +class TinyRoMa(nn.Module): + """ + Implementation of architecture described in + "XFeat: Accelerated Features for Lightweight Image Matching, CVPR 2024." + """ + + def __init__(self, xfeat = None, + freeze_xfeat = True, + sample_mode = "threshold_balanced", + symmetric = False, + exact_softmax = False): + super().__init__() + del xfeat.heatmap_head, xfeat.keypoint_head, xfeat.fine_matcher + if freeze_xfeat: + xfeat.train(False) + self.xfeat = [xfeat]# hide params from ddp + else: + self.xfeat = nn.ModuleList([xfeat]) + self.freeze_xfeat = freeze_xfeat + match_dim = 256 + self.coarse_matcher = nn.Sequential( + BasicLayer(64+64+2, match_dim,), + BasicLayer(match_dim, match_dim,), + BasicLayer(match_dim, match_dim,), + BasicLayer(match_dim, match_dim,), + nn.Conv2d(match_dim, 3, kernel_size=1, bias=True, padding=0)) + fine_match_dim = 64 + self.fine_matcher = nn.Sequential( + BasicLayer(24+24+2, fine_match_dim,), + BasicLayer(fine_match_dim, fine_match_dim,), + BasicLayer(fine_match_dim, fine_match_dim,), + BasicLayer(fine_match_dim, fine_match_dim,), + nn.Conv2d(fine_match_dim, 3, kernel_size=1, bias=True, padding=0),) + self.sample_mode = sample_mode + self.sample_thresh = 0.05 + self.symmetric = symmetric + self.exact_softmax = exact_softmax + + @property + def device(self): + return self.fine_matcher[-1].weight.device + + def preprocess_tensor(self, x): + """ Guarantee that image is divisible by 32 to avoid aliasing artifacts. """ + H, W = x.shape[-2:] + _H, _W = (H//32) * 32, (W//32) * 32 + rh, rw = H/_H, W/_W + + x = F.interpolate(x, (_H, _W), mode='bilinear', align_corners=False) + return x, rh, rw + + def forward_single(self, x): + with torch.inference_mode(self.freeze_xfeat or not self.training): + xfeat = self.xfeat[0] + with torch.no_grad(): + x = x.mean(dim=1, keepdim = True) + x = xfeat.norm(x) + + #main backbone + x1 = xfeat.block1(x) + x2 = xfeat.block2(x1 + xfeat.skip1(x)) + x3 = xfeat.block3(x2) + x4 = xfeat.block4(x3) + x5 = xfeat.block5(x4) + x4 = F.interpolate(x4, (x3.shape[-2], x3.shape[-1]), mode='bilinear') + x5 = F.interpolate(x5, (x3.shape[-2], x3.shape[-1]), mode='bilinear') + feats = xfeat.block_fusion( x3 + x4 + x5 ) + if self.freeze_xfeat: + return x2.clone(), feats.clone() + return x2, feats + + def to_pixel_coordinates(self, coords, H_A, W_A, H_B = None, W_B = None): + if coords.shape[-1] == 2: + return self._to_pixel_coordinates(coords, H_A, W_A) + + if isinstance(coords, (list, tuple)): + kpts_A, kpts_B = coords[0], coords[1] + else: + kpts_A, kpts_B = coords[...,:2], coords[...,2:] + return self._to_pixel_coordinates(kpts_A, H_A, W_A), self._to_pixel_coordinates(kpts_B, H_B, W_B) + + def _to_pixel_coordinates(self, coords, H, W): + kpts = torch.stack((W/2 * (coords[...,0]+1), H/2 * (coords[...,1]+1)),axis=-1) + return kpts + + def pos_embed(self, corr_volume: torch.Tensor): + B, H1, W1, H0, W0 = corr_volume.shape + grid = torch.stack( + torch.meshgrid( + torch.linspace(-1+1/W1,1-1/W1, W1), + torch.linspace(-1+1/H1,1-1/H1, H1), + indexing = "xy"), + dim = -1).float().to(corr_volume).reshape(H1*W1, 2) + down = 4 + if not self.training and not self.exact_softmax: + grid_lr = torch.stack( + torch.meshgrid( + torch.linspace(-1+down/W1,1-down/W1, W1//down), + torch.linspace(-1+down/H1,1-down/H1, H1//down), + indexing = "xy"), + dim = -1).float().to(corr_volume).reshape(H1*W1 //down**2, 2) + cv = corr_volume + best_match = cv.reshape(B,H1*W1,H0,W0).argmax(dim=1) # B, HW, H, W + P_lowres = torch.cat((cv[:,::down,::down].reshape(B,H1*W1 // down**2,H0,W0), best_match[:,None]),dim=1).softmax(dim=1) + pos_embeddings = torch.einsum('bchw,cd->bdhw', P_lowres[:,:-1], grid_lr) + pos_embeddings += P_lowres[:,-1] * grid[best_match].permute(0,3,1,2) + #print("hej") + else: + P = corr_volume.reshape(B,H1*W1,H0,W0).softmax(dim=1) # B, HW, H, W + pos_embeddings = torch.einsum('bchw,cd->bdhw', P, grid) + return pos_embeddings + + def visualize_warp(self, warp, certainty, im_A = None, im_B = None, + im_A_path = None, im_B_path = None, symmetric = True, save_path = None, unnormalize = False): + device = warp.device + H,W2,_ = warp.shape + W = W2//2 if symmetric else W2 + if im_A is None: + from PIL import Image + im_A, im_B = Image.open(im_A_path).convert("RGB"), Image.open(im_B_path).convert("RGB") + if not isinstance(im_A, torch.Tensor): + im_A = im_A.resize((W,H)) + im_B = im_B.resize((W,H)) + x_B = (torch.tensor(np.array(im_B)) / 255).to(device).permute(2, 0, 1) + if symmetric: + x_A = (torch.tensor(np.array(im_A)) / 255).to(device).permute(2, 0, 1) + else: + if symmetric: + x_A = im_A + x_B = im_B + im_A_transfer_rgb = F.grid_sample( + x_B[None], warp[:,:W, 2:][None], mode="bilinear", align_corners=False + )[0] + if symmetric: + im_B_transfer_rgb = F.grid_sample( + x_A[None], warp[:, W:, :2][None], mode="bilinear", align_corners=False + )[0] + warp_im = torch.cat((im_A_transfer_rgb,im_B_transfer_rgb),dim=2) + white_im = torch.ones((H,2*W),device=device) + else: + warp_im = im_A_transfer_rgb + white_im = torch.ones((H, W), device = device) + vis_im = certainty * warp_im + (1 - certainty) * white_im + if save_path is not None: + from romatch.utils import tensor_to_pil + tensor_to_pil(vis_im, unnormalize=unnormalize).save(save_path) + return vis_im + + def corr_volume(self, feat0, feat1): + """ + input: + feat0 -> torch.Tensor(B, C, H, W) + feat1 -> torch.Tensor(B, C, H, W) + return: + corr_volume -> torch.Tensor(B, H, W, H, W) + """ + B, C, H0, W0 = feat0.shape + B, C, H1, W1 = feat1.shape + feat0 = feat0.view(B, C, H0*W0) + feat1 = feat1.view(B, C, H1*W1) + corr_volume = torch.einsum('bci,bcj->bji', feat0, feat1).reshape(B, H1, W1, H0 , W0)/math.sqrt(C) #16*16*16 + return corr_volume + + @torch.inference_mode() + def match_from_path(self, im0_path, im1_path): + device = self.device + im0 = ToTensor()(Image.open(im0_path))[None].to(device) + im1 = ToTensor()(Image.open(im1_path))[None].to(device) + return self.match(im0, im1, batched = False) + + @torch.inference_mode() + def match(self, im0, im1, *args, batched = True): + # stupid + if isinstance(im0, (str, Path)): + return self.match_from_path(im0, im1) + elif isinstance(im0, Image.Image): + batched = False + device = self.device + im0 = ToTensor()(im0)[None].to(device) + im1 = ToTensor()(im1)[None].to(device) + + B,C,H0,W0 = im0.shape + B,C,H1,W1 = im1.shape + self.train(False) + corresps = self.forward({"im_A":im0, "im_B":im1}) + #return 1,1 + flow = F.interpolate( + corresps[4]["flow"], + size = (H0, W0), + mode = "bilinear", align_corners = False).permute(0,2,3,1).reshape(B,H0,W0,2) + grid = torch.stack( + torch.meshgrid( + torch.linspace(-1+1/W0,1-1/W0, W0), + torch.linspace(-1+1/H0,1-1/H0, H0), + indexing = "xy"), + dim = -1).float().to(flow.device).expand(B, H0, W0, 2) + + certainty = F.interpolate(corresps[4]["certainty"], size = (H0,W0), mode = "bilinear", align_corners = False) + warp, cert = torch.cat((grid, flow), dim = -1), certainty[:,0].sigmoid() + if batched: + return warp, cert + else: + return warp[0], cert[0] + + def sample( + self, + matches, + certainty, + num=5_000, + ): + H,W,_ = matches.shape + if "threshold" in self.sample_mode: + upper_thresh = self.sample_thresh + certainty = certainty.clone() + certainty[certainty > upper_thresh] = 1 + matches, certainty = ( + matches.reshape(-1, 4), + certainty.reshape(-1), + ) + expansion_factor = 4 if "balanced" in self.sample_mode else 1 + good_samples = torch.multinomial(certainty, + num_samples = min(expansion_factor*num, len(certainty)), + replacement=False) + good_matches, good_certainty = matches[good_samples], certainty[good_samples] + if "balanced" not in self.sample_mode: + return good_matches, good_certainty + use_half = True if matches.device.type == "cuda" else False + down = 1 if matches.device.type == "cuda" else 8 + density = kde(good_matches, std=0.1, half = use_half, down = down) + p = 1 / (density+1) + p[density < 10] = 1e-7 # Basically should have at least 10 perfect neighbours, or around 100 ok ones + balanced_samples = torch.multinomial(p, + num_samples = min(num,len(good_certainty)), + replacement=False) + return good_matches[balanced_samples], good_certainty[balanced_samples] + + + def forward(self, batch): + """ + input: + x -> torch.Tensor(B, C, H, W) grayscale or rgb images + return: + + """ + im0 = batch["im_A"] + im1 = batch["im_B"] + corresps = {} + im0, rh0, rw0 = self.preprocess_tensor(im0) + im1, rh1, rw1 = self.preprocess_tensor(im1) + B, C, H0, W0 = im0.shape + B, C, H1, W1 = im1.shape + to_normalized = torch.tensor((2/W1, 2/H1, 1)).to(im0.device)[None,:,None,None] + + if im0.shape[-2:] == im1.shape[-2:]: + x = torch.cat([im0, im1], dim=0) + x = self.forward_single(x) + feats_x0_c, feats_x1_c = x[1].chunk(2) + feats_x0_f, feats_x1_f = x[0].chunk(2) + else: + feats_x0_f, feats_x0_c = self.forward_single(im0) + feats_x1_f, feats_x1_c = self.forward_single(im1) + corr_volume = self.corr_volume(feats_x0_c, feats_x1_c) + coarse_warp = self.pos_embed(corr_volume) + coarse_matches = torch.cat((coarse_warp, torch.zeros_like(coarse_warp[:,-1:])), dim=1) + feats_x1_c_warped = F.grid_sample(feats_x1_c, coarse_matches.permute(0, 2, 3, 1)[...,:2], mode = 'bilinear', align_corners = False) + coarse_matches_delta = self.coarse_matcher(torch.cat((feats_x0_c, feats_x1_c_warped, coarse_warp), dim=1)) + coarse_matches = coarse_matches + coarse_matches_delta * to_normalized + corresps[8] = {"flow": coarse_matches[:,:2], "certainty": coarse_matches[:,2:]} + coarse_matches_up = F.interpolate(coarse_matches, size = feats_x0_f.shape[-2:], mode = "bilinear", align_corners = False) + coarse_matches_up_detach = coarse_matches_up.detach()#note the detach + feats_x1_f_warped = F.grid_sample(feats_x1_f, coarse_matches_up_detach.permute(0, 2, 3, 1)[...,:2], mode = 'bilinear', align_corners = False) + fine_matches_delta = self.fine_matcher(torch.cat((feats_x0_f, feats_x1_f_warped, coarse_matches_up_detach[:,:2]), dim=1)) + fine_matches = coarse_matches_up_detach+fine_matches_delta * to_normalized + corresps[4] = {"flow": fine_matches[:,:2], "certainty": fine_matches[:,2:]} + return corresps \ No newline at end of file diff --git a/third_party/RoMa/roma/models/transformer/__init__.py b/third_party/RoMa/romatch/models/transformer/__init__.py similarity index 97% rename from third_party/RoMa/roma/models/transformer/__init__.py rename to third_party/RoMa/romatch/models/transformer/__init__.py index c93008ecdaab3fa19d7166b213f8d4f664bf65d5..17a1f7df8829bb54f55109780e71543933992c0b 100644 --- a/third_party/RoMa/roma/models/transformer/__init__.py +++ b/third_party/RoMa/romatch/models/transformer/__init__.py @@ -2,7 +2,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -from roma.utils.utils import get_grid +from romatch.utils.utils import get_grid from .layers.block import Block from .layers.attention import MemEffAttention from .dinov2 import vit_large diff --git a/third_party/RoMa/roma/models/transformer/dinov2.py b/third_party/RoMa/romatch/models/transformer/dinov2.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/dinov2.py rename to third_party/RoMa/romatch/models/transformer/dinov2.py diff --git a/third_party/RoMa/roma/models/transformer/layers/__init__.py b/third_party/RoMa/romatch/models/transformer/layers/__init__.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/__init__.py rename to third_party/RoMa/romatch/models/transformer/layers/__init__.py diff --git a/third_party/RoMa/roma/models/transformer/layers/attention.py b/third_party/RoMa/romatch/models/transformer/layers/attention.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/attention.py rename to third_party/RoMa/romatch/models/transformer/layers/attention.py diff --git a/third_party/RoMa/roma/models/transformer/layers/block.py b/third_party/RoMa/romatch/models/transformer/layers/block.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/block.py rename to third_party/RoMa/romatch/models/transformer/layers/block.py diff --git a/third_party/RoMa/roma/models/transformer/layers/dino_head.py b/third_party/RoMa/romatch/models/transformer/layers/dino_head.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/dino_head.py rename to third_party/RoMa/romatch/models/transformer/layers/dino_head.py diff --git a/third_party/RoMa/roma/models/transformer/layers/drop_path.py b/third_party/RoMa/romatch/models/transformer/layers/drop_path.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/drop_path.py rename to third_party/RoMa/romatch/models/transformer/layers/drop_path.py diff --git a/third_party/RoMa/roma/models/transformer/layers/layer_scale.py b/third_party/RoMa/romatch/models/transformer/layers/layer_scale.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/layer_scale.py rename to third_party/RoMa/romatch/models/transformer/layers/layer_scale.py diff --git a/third_party/RoMa/roma/models/transformer/layers/mlp.py b/third_party/RoMa/romatch/models/transformer/layers/mlp.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/mlp.py rename to third_party/RoMa/romatch/models/transformer/layers/mlp.py diff --git a/third_party/RoMa/roma/models/transformer/layers/patch_embed.py b/third_party/RoMa/romatch/models/transformer/layers/patch_embed.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/patch_embed.py rename to third_party/RoMa/romatch/models/transformer/layers/patch_embed.py diff --git a/third_party/RoMa/roma/models/transformer/layers/swiglu_ffn.py b/third_party/RoMa/romatch/models/transformer/layers/swiglu_ffn.py similarity index 100% rename from third_party/RoMa/roma/models/transformer/layers/swiglu_ffn.py rename to third_party/RoMa/romatch/models/transformer/layers/swiglu_ffn.py diff --git a/third_party/RoMa/roma/train/__init__.py b/third_party/RoMa/romatch/train/__init__.py similarity index 100% rename from third_party/RoMa/roma/train/__init__.py rename to third_party/RoMa/romatch/train/__init__.py diff --git a/third_party/RoMa/roma/train/train.py b/third_party/RoMa/romatch/train/train.py similarity index 90% rename from third_party/RoMa/roma/train/train.py rename to third_party/RoMa/romatch/train/train.py index 5556f7ebf9b6378e1395c125dde093f5e55e7141..7cb02ed1e816fd39f174f76ec15bce49ae2a3da8 100644 --- a/third_party/RoMa/roma/train/train.py +++ b/third_party/RoMa/romatch/train/train.py @@ -1,6 +1,6 @@ from tqdm import tqdm -from roma.utils.utils import to_cuda -import roma +from romatch.utils.utils import to_cuda +import romatch import torch import wandb @@ -17,8 +17,8 @@ def log_param_statistics(named_parameters, norm_type = 2): total_grad_norm = torch.norm(grad_norms, norm_type) if torch.any(nans_or_infs): print(f"These params have nan or inf grads: {nan_inf_names}") - wandb.log({"grad_norm": total_grad_norm.item()}, step = roma.GLOBAL_STEP) - wandb.log({"param_norm": param_norm.item()}, step = roma.GLOBAL_STEP) + wandb.log({"grad_norm": total_grad_norm.item()}, step = romatch.GLOBAL_STEP) + wandb.log({"param_norm": param_norm.item()}, step = romatch.GLOBAL_STEP) def train_step(train_batch, model, objective, optimizer, grad_scaler, grad_clip_norm = 1.,**kwargs): optimizer.zero_grad() @@ -30,17 +30,17 @@ def train_step(train_batch, model, objective, optimizer, grad_scaler, grad_clip_ torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip_norm) # what should max norm be? grad_scaler.step(optimizer) grad_scaler.update() - wandb.log({"grad_scale": grad_scaler._scale.item()}, step = roma.GLOBAL_STEP) + wandb.log({"grad_scale": grad_scaler._scale.item()}, step = romatch.GLOBAL_STEP) if grad_scaler._scale < 1.: grad_scaler._scale = torch.tensor(1.).to(grad_scaler._scale) - roma.GLOBAL_STEP = roma.GLOBAL_STEP + roma.STEP_SIZE # increment global step + romatch.GLOBAL_STEP = romatch.GLOBAL_STEP + romatch.STEP_SIZE # increment global step return {"train_out": out, "train_loss": l.item()} def train_k_steps( - n_0, k, dataloader, model, objective, optimizer, lr_scheduler, grad_scaler, progress_bar=True, grad_clip_norm = 1., warmup = None, ema_model = None, + n_0, k, dataloader, model, objective, optimizer, lr_scheduler, grad_scaler, progress_bar=True, grad_clip_norm = 1., warmup = None, ema_model = None, pbar_n_seconds = 1, ): - for n in tqdm(range(n_0, n_0 + k), disable=(not progress_bar) or roma.RANK > 0): + for n in tqdm(range(n_0, n_0 + k), disable=(not progress_bar) or romatch.RANK > 0, mininterval=pbar_n_seconds): batch = next(dataloader) model.train(True) batch = to_cuda(batch) diff --git a/third_party/RoMa/roma/utils/__init__.py b/third_party/RoMa/romatch/utils/__init__.py similarity index 100% rename from third_party/RoMa/roma/utils/__init__.py rename to third_party/RoMa/romatch/utils/__init__.py diff --git a/third_party/RoMa/romatch/utils/kde.py b/third_party/RoMa/romatch/utils/kde.py new file mode 100644 index 0000000000000000000000000000000000000000..46ed2e5e106bbca93e703f39f3ad3af350666e34 --- /dev/null +++ b/third_party/RoMa/romatch/utils/kde.py @@ -0,0 +1,13 @@ +import torch + + +def kde(x, std = 0.1, half = True, down = None): + # use a gaussian kernel to estimate density + if half: + x = x.half() # Do it in half precision TODO: remove hardcoding + if down is not None: + scores = (-torch.cdist(x,x[::down])**2/(2*std**2)).exp() + else: + scores = (-torch.cdist(x,x)**2/(2*std**2)).exp() + density = scores.sum(dim=-1) + return density \ No newline at end of file diff --git a/third_party/RoMa/roma/utils/local_correlation.py b/third_party/RoMa/romatch/utils/local_correlation.py similarity index 100% rename from third_party/RoMa/roma/utils/local_correlation.py rename to third_party/RoMa/romatch/utils/local_correlation.py diff --git a/third_party/RoMa/roma/utils/transforms.py b/third_party/RoMa/romatch/utils/transforms.py similarity index 100% rename from third_party/RoMa/roma/utils/transforms.py rename to third_party/RoMa/romatch/utils/transforms.py diff --git a/third_party/RoMa/roma/utils/utils.py b/third_party/RoMa/romatch/utils/utils.py similarity index 100% rename from third_party/RoMa/roma/utils/utils.py rename to third_party/RoMa/romatch/utils/utils.py diff --git a/third_party/RoMa/setup.py b/third_party/RoMa/setup.py index fe2e6dc4be62254f702e34422e07468b00195dd2..7ec18f3bbb71b85d943fdfeed3ed5c47033aebbc 100644 --- a/third_party/RoMa/setup.py +++ b/third_party/RoMa/setup.py @@ -1,8 +1,8 @@ from setuptools import setup, find_packages setup( - name="roma", - packages=find_packages(include=("roma*",)), + name="romatch", + packages=find_packages(include=("romatch*",)), version="0.0.1", author="Johan Edstedt", install_requires=open("requirements.txt", "r").read().split("\n"), diff --git a/third_party/dust3r/.gitignore b/third_party/dust3r/.gitignore index 194e236cbd708160926c3513b4232285eb47b029..0eb590aa03e6840537cce148886a74de6dcce096 100644 --- a/third_party/dust3r/.gitignore +++ b/third_party/dust3r/.gitignore @@ -3,6 +3,7 @@ checkpoints/ # Byte-compiled / optimized / DLL files __pycache__/ +__pycache__* *.py[cod] *$py.class diff --git a/third_party/dust3r/README.md b/third_party/dust3r/README.md index de70cb506bd9400aa81bd8f6d8add0ac58e563ef..013646478823a1ac77f3c70603abb35650b58304 100644 --- a/third_party/dust3r/README.md +++ b/third_party/dust3r/README.md @@ -3,6 +3,8 @@ Official implementation of `DUSt3R: Geometric 3D Vision Made Easy` [[Project page](https://dust3r.europe.naverlabs.com/)], [[DUSt3R arxiv](https://arxiv.org/abs/2312.14132)] +> :warning: **We have removed the checkpoints temporarily**: We apologize for that! + ![Example of reconstruction from two images](assets/pipeline1.jpg) ![High level overview of DUSt3R capabilities](assets/dust3r_archi.jpg) @@ -81,6 +83,7 @@ cd ../../../ ``` ### Checkpoints +> :warning: **We have removed the checkpoints temporarily**: We apologize for that! You can obtain the checkpoints by two ways: @@ -92,16 +95,18 @@ You can obtain the checkpoints by two ways: |-------------|----------------------|------|---------|---------| | [`DUSt3R_ViTLarge_BaseDecoder_224_linear.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_224_linear.pth) | 224x224 | Linear | ViT-L | ViT-B | | [`DUSt3R_ViTLarge_BaseDecoder_512_linear.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_linear.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | Linear | ViT-L | ViT-B | -| [`DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | DPT | ViT-L | ViT-B | +| [`DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`]() | 512x384, 512x336, 512x288, 512x256, 512x160 | DPT | ViT-L | ViT-B | You can check the hyperparameters we used to train these models in the [section: Our Hyperparameters](#our-hyperparameters) To download a specific model, for example `DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`: ```bash mkdir -p checkpoints/ -wget https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth -P checkpoints/ +wget TODO -P checkpoints/ ``` +For the checkpoints, make sure to agree to the license of all the public training datasets and base checkpoints we used, in addition to CC-BY-NC-SA 4.0. Again, see [section: Our Hyperparameters](#our-hyperparameters) for details. + ### Interactive demo In this demo, you should be able run DUSt3R on your machine to reconstruct a scene. @@ -317,9 +322,9 @@ We didn't release the training datasets, but here are the commands we used for t ```bash # NOTE: ROOT path omitted for datasets # 224 linear -torchrun --nproc_per_node 4 train.py \ - --train_dataset=" + 100_000 @ Habitat512(1_000_000, split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ BlendedMVS(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ MegaDepthDense(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ ARKitScenes(aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ Co3d_v3(split='train', aug_crop=16, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ ScanNetpp(split='train', aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ Waymo(aug_crop=128, resolution=224, transform=ColorJitter) " \ - --test_dataset=" Habitat512(1_000, split='val', resolution=224, seed=777) + 1_000 @ BlendedMVS(split='val', resolution=224, seed=777) + 1_000 @ MegaDepthDense(split='val', resolution=224, seed=777) + 1_000 @ Co3d_v3(split='test', mask_bg='rand', resolution=224, seed=777) " \ +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 100_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ BlendedMVS(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ MegaDepth(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ ARKitScenes(aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ ScanNetpp(split='train', aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=224, transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=224, seed=777) + 1_000 @ BlendedMVS(split='val', resolution=224, seed=777) + 1_000 @ MegaDepth(split='val', resolution=224, seed=777) + 1_000 @ Co3d(split='test', mask_bg='rand', resolution=224, seed=777) " \ --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', img_size=(224, 224), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ @@ -330,25 +335,25 @@ torchrun --nproc_per_node 4 train.py \ # 512 linear torchrun --nproc_per_node 8 train.py \ - --train_dataset=" + 10_000 @ Habitat512(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepthDense(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d_v3(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Waymo(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ - --test_dataset=" Habitat512(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepthDense(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d_v3(split='test', resolution=(512,384), seed=777) " \ + --train_dataset=" + 10_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepth(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepth(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d(split='test', resolution=(512,384), seed=777) " \ --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ --pretrained="checkpoints/dust3r_224/checkpoint-best.pth" \ - --lr=0.0001 --min_lr=1e-06 --warmup_epochs=20 --epochs=200 --batch_size=4 --accum_iter=2 \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=20 --epochs=100 --batch_size=4 --accum_iter=2 \ --save_freq=10 --keep_freq=10 --eval_freq=1 --print_freq=10 \ --output_dir="checkpoints/dust3r_512" # 512 dpt torchrun --nproc_per_node 8 train.py \ - --train_dataset=" + 10_000 @ Habitat512(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepthDense(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d_v3(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Waymo(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ - --test_dataset=" Habitat512(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepthDense(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d_v3(split='test', resolution=(512,384), seed=777) " \ + --train_dataset=" + 10_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepth(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepth(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d(split='test', resolution=(512,384), seed=777) " \ --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='dpt', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ --pretrained="checkpoints/dust3r_512/checkpoint-best.pth" \ - --lr=0.0001 --min_lr=1e-06 --warmup_epochs=15 --epochs=90 --batch_size=2 --accum_iter=4 \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=15 --epochs=90 --batch_size=4 --accum_iter=2 \ --save_freq=5 --keep_freq=10 --eval_freq=1 --print_freq=10 \ --output_dir="checkpoints/dust3r_512dpt" diff --git a/third_party/dust3r/croco/.gitignore b/third_party/dust3r/croco/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0eb590aa03e6840537cce148886a74de6dcce096 --- /dev/null +++ b/third_party/dust3r/croco/.gitignore @@ -0,0 +1,133 @@ +data/ +checkpoints/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +__pycache__* +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/third_party/dust3r/dust3r/cloud_opt/base_opt.py b/third_party/dust3r/dust3r/cloud_opt/base_opt.py index 7038bc37163ec8447712f96cc29b77c4d188b0ad..62c1ea0666c058bb944176e9b821754f32ff90da 100644 --- a/third_party/dust3r/dust3r/cloud_opt/base_opt.py +++ b/third_party/dust3r/dust3r/cloud_opt/base_opt.py @@ -79,6 +79,8 @@ class BasePCOptimizer (nn.Module): self.conf_i = NoGradParamDict({ij: pred1_conf[n] for n, ij in enumerate(self.str_edges)}) self.conf_j = NoGradParamDict({ij: pred2_conf[n] for n, ij in enumerate(self.str_edges)}) self.im_conf = self._compute_img_conf(pred1_conf, pred2_conf) + for i in range(len(self.im_conf)): + self.im_conf[i].requires_grad = False # pairwise pose parameters self.base_scale = base_scale @@ -364,12 +366,12 @@ def global_alignment_loop(net, lr=0.01, niter=300, schedule='cosine', lr_min=1e- if verbose: with tqdm.tqdm(total=niter) as bar: while bar.n < bar.total: - loss = global_alignment_iter(net, bar.n, niter, lr_base, lr_min, optimizer, schedule) + loss, lr = global_alignment_iter(net, bar.n, niter, lr_base, lr_min, optimizer, schedule) bar.set_postfix_str(f'{lr=:g} loss={loss:g}') bar.update() else: for n in range(niter): - loss = global_alignment_iter(net, n, niter, lr_base, lr_min, optimizer, schedule) + loss, _ = global_alignment_iter(net, n, niter, lr_base, lr_min, optimizer, schedule) return loss @@ -387,4 +389,4 @@ def global_alignment_iter(net, cur_iter, niter, lr_base, lr_min, optimizer, sche loss.backward() optimizer.step() - return float(loss) + return float(loss), lr diff --git a/third_party/dust3r/dust3r/inference.py b/third_party/dust3r/dust3r/inference.py index a9e668735195531cbca04455fee0b73057db4d4e..95a7eaaa778bb8c6ec869635670a939da00018b5 100644 --- a/third_party/dust3r/dust3r/inference.py +++ b/third_party/dust3r/dust3r/inference.py @@ -6,9 +6,9 @@ # -------------------------------------------------------- import tqdm import torch -from .utils.device import to_cpu, collate_with_cat -from .utils.misc import invalid_to_nans -from .utils.geometry import depthmap_to_pts3d, geotrf +from dust3r.utils.device import to_cpu, collate_with_cat +from dust3r.utils.misc import invalid_to_nans +from dust3r.utils.geometry import depthmap_to_pts3d, geotrf def _interleave_imgs(img1, img2):