Add SFace for face recognition (#3)

* SFace and demo impl

* benchmark support and results for SFace

README.md

@@ -17,7 +17,7 @@ Hardware Setup:
 
 ***Important Notes***:
 - The time data that shown on the following tables presents the time elapsed from preprocess (resize is excluded), to a forward pass of a network, and postprocess to get final results.
-- The time data that shown on the following tables is averaged from a 100-time run.
+- The time data that shown on the following tables is the median of benchmark runs.
 - View [benchmark/config](./benchmark/config) for more details on benchmarking different models.
 
 <!--
@@ -29,9 +29,10 @@ Hardware Setup:
 -->
 | Model | Input Size | CPU x86_64 (ms) | CPU ARM (ms) |
 |-------|------------|-----------------|--------------|
-| [YuNet](./models/face_detection_yunet) | 160x120 | 2.17   | 8.87    |
-| [DB](./models/text_detection_db)       | 640x480 | 148.65 | 2759.88 |
-| [CRNN](./models/text_recognition_crnn) | 100x32  | 23.23  | 235.87  |
+| [YuNet](./models/face_detection_yunet)   | 160x120 | 2.35   | 8.72    |
+| [DB](./models/text_detection_db)         | 640x480 | 137.38 | 2780.78 |
+| [CRNN](./models/text_recognition_crnn)   | 100x32  | 50.21  | 234.32  |
+| [SFace](./models/face_recognition_sface) | 112x112 | 8.69 | 96.79 |
 
 
 ## License

benchmark/benchmark.py

@@ -78,7 +78,7 @@ class Data:
     def _load_label(self):
         labels = dict.fromkeys(self._files, None)
         for filename in self._files:
-            labels[filename] = np.loadtxt(os.path.join(self._path, '{}.txt'.format(filename[:-4])))
+            labels[filename] = np.loadtxt(os.path.join(self._path, '{}.txt'.format(filename[:-4])), ndmin=2)
         return labels
 
     def __getitem__(self, idx):

benchmark/config/face_detection_yunet.yaml

@@ -1,7 +1,7 @@
 Benchmark:
   name: "Face Detection Benchmark"
   data:
-    path: "benchmark/data/face"
+    path: "benchmark/data/face/detection"
     files: ["group.jpg", "concerts.jpg", "dance.jpg"]
   metric:
     sizes: # [[w1, h1], ...], Omit to run at original scale

benchmark/config/face_recognition_sface.yaml

@@ -0,0 +1,17 @@
+Benchmark:
+  name: "Face Recognition Benchmark"
+  data:
+    path: "benchmark/data/face/recognition"
+    files: ["Aaron_Tippin_0001.jpg", "Alvaro_Uribe_0028.jpg", "Alvaro_Uribe_0029.jpg", "Jose_Luis_Rodriguez_Zapatero_0001.jpg"]
+    useLabel: True
+  metric: # 'sizes' is omitted since this model requires input of fixed size
+    warmup: 3
+    repeat: 10
+    batchSize: 1
+    reduction: 'median'
+  backend: "default"
+  target: "cpu"
+
+Model:
+  name: "SFace"
+  modelPath: "models/face_recognition_sface/face_recognition_sface.onnx"

models/__init__.py

@@ -1,6 +1,7 @@
 from .face_detection_yunet.yunet import YuNet
 from .text_detection_db.db import DB
 from .text_recognition_crnn.crnn import CRNN
+from .face_recognition_sface.sface import SFace
 
 class Registery:
     def __init__(self, name):
@@ -16,4 +17,5 @@ class Registery:
 MODELS = Registery('Models')
 MODELS.register(YuNet)
 MODELS.register(DB)
-MODELS.register(CRNN)
+MODELS.register(CRNN)
+MODELS.register(SFace)

models/face_detection_yunet/demo.py

@@ -77,8 +77,8 @@ if __name__ == '__main__':
         # Print results
         print('{} faces detected.'.format(results.shape[0]))
         for idx, det in enumerate(results):
-            print('{}: [{:.0f}, {:.0f}] [{:.0f}, {:.0f}], {:.2f}'.format(
-                idx, det[0], det[1], det[2], det[3], det[-1])
+            print('{}: {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f}'.format(
+                idx, *det[:-1])
             )
 
         # Draw results on the input image

models/face_recognition_sface/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

models/face_recognition_sface/README.md

@@ -0,0 +1,27 @@
+# SFace
+
+SFace: Sigmoid-Constrained Hypersphere Loss for Robust Face Recognition
+
+SFace is contributed by [Yaoyao Zhong](https://github.com/zhongyy/SFace). [face_recognition_sface.onnx](./face_recognition_sface.onnx) is converted from the model from https://github.com/zhongyy/SFace thanks to [Chengrui Wang](https://github.com/crywang).
+
+Note:
+- There is [a PR for OpenCV adding this model](https://github.com/opencv/opencv/pull/20422) to work with OpenCV DNN in C++ implementation.
+- Support 5-landmark warp for now.
+- `demo.py` requires [../face_detection_yunet](../face_detection_yunet) to run.
+
+## Demo
+
+Run the following command to try the demo:
+```shell
+# recognize on images
+python demo.py --input1 /path/to/image1 --input2 /path/to/image2
+```
+
+## License
+
+All files in this directory are licensed under [Apache 2.0 License](./LICENSE).
+
+## Reference
+
+- https://ieeexplore.ieee.org/document/9318547
+- https://github.com/zhongyy/SFace

models/face_recognition_sface/demo.py

@@ -0,0 +1,71 @@
+# This file is part of OpenCV Zoo project.
+# It is subject to the license terms in the LICENSE file found in the same directory.
+#
+# Copyright (C) 2021, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved.
+# Third party copyrights are property of their respective owners.
+
+import sys
+import argparse
+
+import numpy as np
+import cv2 as cv
+
+from sface import SFace
+
+sys.path.append('../face_detection_yunet')
+from yunet import YuNet
+
+def str2bool(v):
+    if v.lower() in ['on', 'yes', 'true', 'y', 't']:
+        return True
+    elif v.lower() in ['off', 'no', 'false', 'n', 'f']:
+        return False
+    else:
+        raise NotImplementedError
+
+parser = argparse.ArgumentParser(
+    description="SFace: Sigmoid-Constrained Hypersphere Loss for Robust Face Recognition (https://ieeexplore.ieee.org/document/9318547)")
+parser.add_argument('--input1', '-i1', type=str, help='Path to the input image 1.')
+parser.add_argument('--input2', '-i2', type=str, help='Path to the input image 2.')
+parser.add_argument('--model', '-m', type=str, default='face_recognition_sface.onnx', help='Path to the model.')
+parser.add_argument('--dis_type', type=int, choices=[0, 1], default=0, help='Distance type. \'0\': cosine, \'1\': norm_l1.')
+parser.add_argument('--save', '-s', type=str, default=False, help='Set true to save results. This flag is invalid when using camera.')
+parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.')
+args = parser.parse_args()
+
+if __name__ == '__main__':
+    # Instantiate SFace for face recognition
+    recognizer = SFace(modelPath=args.model)
+    # Instantiate YuNet for face detection
+    detector = YuNet(modelPath='../face_detection_yunet/face_detection_yunet.onnx',
+                     inputSize=[320, 320],
+                     confThreshold=0.9,
+                     nmsThreshold=0.3,
+                     topK=5000,
+                     keepTopK=750)
+
+    img1 = cv.imread(args.input1)
+    img2 = cv.imread(args.input2)
+
+    # Detect faces
+    detector.setInputSize([img1.shape[1], img1.shape[0]])
+    face1 = detector.infer(img1)
+    assert face1.shape[0] > 0, 'Cannot find a face in {}'.format(args.input1)
+    detector.setInputSize([img2.shape[1], img2.shape[0]])
+    face2 = detector.infer(img2)
+    assert face2.shape[0] > 0, 'Cannot find a face in {}'.format(args.input2)
+
+    # Match
+    distance = recognizer.match(img1, face1[0][:-1], img2, face2[0][:-1], args.dis_type)
+    print(distance)
+    if args.dis_type == 0:
+        dis_type = 'Cosine'
+        threshold = 0.363
+        result = 'same identity' if distance >= threshold else 'different identity'
+    elif args.dis_type == 1:
+        dis_type = 'Norm-L2'
+        threshold = 1.128
+        result = 'same identity' if distance <= threshold else 'different identity'
+    else:
+        raise NotImplementedError()
+    print('Using {} distance, threshold {}: {}.'.format(dis_type, threshold, result))

models/face_recognition_sface/sface.py

@@ -0,0 +1,165 @@
+# This file is part of OpenCV Zoo project.
+# It is subject to the license terms in the LICENSE file found in the same directory.
+#
+# Copyright (C) 2021, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved.
+# Third party copyrights are property of their respective owners.
+
+import numpy as np
+import cv2 as cv
+
+from _testcapi import FLT_MIN
+
+class SFace:
+    def __init__(self, modelPath):
+        self._model = cv.dnn.readNet(modelPath)
+        self._input_size = [112, 112]
+        self._dst = np.array([
+            [38.2946, 51.6963],
+            [73.5318, 51.5014],
+            [56.0252, 71.7366],
+            [41.5493, 92.3655],
+            [70.7299, 92.2041]
+        ], dtype=np.float32)
+        self._dst_mean = np.array([56.0262, 71.9008], dtype=np.float32)
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def setBackend(self, backend_id):
+        self._model.setPreferableBackend(backend_id)
+
+    def setTarget(self, target_id):
+        self._model.setPreferableTarget(target_id)
+
+    def _preprocess(self, image, bbox):
+        aligned_image = self._alignCrop(image, bbox)
+        return cv.dnn.blobFromImage(aligned_image)
+
+    def infer(self, image, bbox):
+        # Preprocess
+        inputBlob = self._preprocess(image, bbox)
+
+        # Forward
+        self._model.setInput(inputBlob)
+        outputBlob = self._model.forward()
+
+        # Postprocess
+        results = self._postprocess(outputBlob)
+
+        return results
+
+    def _postprocess(self, outputBlob):
+        return outputBlob / cv.norm(outputBlob)
+
+    def match(self, image1, face1, image2, face2, dis_type=0):
+        feature1 = self.infer(image1, face1)
+        feature2 = self.infer(image2, face2)
+
+        if dis_type == 0: # COSINE
+            return np.sum(feature1 * feature2)
+        elif dis_type == 1: # NORM_L2
+            return cv.norm(feature1, feature2)
+        else:
+            raise NotImplementedError()
+
+    def _alignCrop(self, image, face):
+        # Retrieve landmarks
+        if face.shape[-1] == (4 + 5 * 2):
+            landmarks = face[4:].reshape(5, 2)
+        else:
+            raise NotImplementedError()
+        warp_mat = self._getSimilarityTransformMatrix(landmarks)
+        aligned_image = cv.warpAffine(image, warp_mat, self._input_size, flags=cv.INTER_LINEAR)
+        return aligned_image
+
+    def _getSimilarityTransformMatrix(self, src):
+        # compute the mean of src and dst
+        src_mean = np.array([np.mean(src[:, 0]), np.mean(src[:, 1])], dtype=np.float32)
+        dst_mean = np.array([56.0262, 71.9008], dtype=np.float32)
+        # subtract the means from src and dst
+        src_demean = src.copy()
+        src_demean[:, 0] = src_demean[:, 0] - src_mean[0]
+        src_demean[:, 1] = src_demean[:, 1] - src_mean[1]
+        dst_demean = self._dst.copy()
+        dst_demean[:, 0] = dst_demean[:, 0] - dst_mean[0]
+        dst_demean[:, 1] = dst_demean[:, 1] - dst_mean[1]
+
+        A = np.array([[0., 0.], [0., 0.]], dtype=np.float64)
+        for i in range(5):
+            A[0][0] += dst_demean[i][0] * src_demean[i][0]
+            A[0][1] += dst_demean[i][0] * src_demean[i][1]
+            A[1][0] += dst_demean[i][1] * src_demean[i][0]
+            A[1][1] += dst_demean[i][1] * src_demean[i][1]
+        A = A / 5
+
+        d = np.array([1.0, 1.0], dtype=np.float64)
+        if A[0][0] * A[1][1] - A[0][1] * A[1][0] < 0:
+            d[1] = -1
+
+        T = np.array([
+            [1.0, 0.0, 0.0],
+            [0.0, 1.0, 0.0],
+            [0.0, 0.0, 1.0]
+        ], dtype=np.float64)
+
+        s, u, vt = cv.SVDecomp(A)
+        smax = s[0][0] if s[0][0] > s[1][0] else s[1][0]
+        tol = smax * 2 * FLT_MIN
+        rank = int(0)
+        if s[0][0] > tol:
+            rank += 1
+        if s[1][0] > tol:
+            rank += 1
+        det_u = u[0][0] * u[1][1] - u[0][1] * u[1][0]
+        det_vt = vt[0][0] * vt[1][1] - vt[0][1] * vt[1][0]
+        if rank == 1:
+            if det_u * det_vt > 0:
+                uvt = np.matmul(u, vt)
+                T[0][0] = uvt[0][0]
+                T[0][1] = uvt[0][1]
+                T[1][0] = uvt[1][0]
+                T[1][1] = uvt[1][1]
+            else:
+                temp = d[1]
+                d[1] = -1
+                D = np.array([[d[0], 0.0], [0.0, d[1]]], dtype=np.float64)
+                Dvt = np.matmul(D, vt)
+                uDvt = np.matmul(u, Dvt)
+                T[0][0] = uDvt[0][0]
+                T[0][1] = uDvt[0][1]
+                T[1][0] = uDvt[1][0]
+                T[1][1] = uDvt[1][1]
+                d[1] = temp
+        else:
+            D = np.array([[d[0], 0.0], [0.0, d[1]]], dtype=np.float64)
+            Dvt = np.matmul(D, vt)
+            uDvt = np.matmul(u, Dvt)
+            T[0][0] = uDvt[0][0]
+            T[0][1] = uDvt[0][1]
+            T[1][0] = uDvt[1][0]
+            T[1][1] = uDvt[1][1]
+
+        var1 = 0.0
+        var2 = 0.0
+        for i in range(5):
+            var1 += src_demean[i][0] * src_demean[i][0]
+            var2 += src_demean[i][1] * src_demean[i][1]
+        var1 /= 5
+        var2 /= 5
+
+        scale = 1.0 / (var1 + var2) * (s[0][0] * d[0] + s[1][0] * d[1])
+        TS = [
+            T[0][0] * src_mean[0] + T[0][1] * src_mean[1],
+            T[1][0] * src_mean[0] + T[1][1] * src_mean[1]
+        ]
+        T[0][2] = dst_mean[0] - scale * TS[0]
+        T[1][2] = dst_mean[1] - scale * TS[1]
+        T[0][0] *= scale
+        T[0][1] *= scale
+        T[1][0] *= scale
+        T[1][1] *= scale
+        return np.array([
+            [T[0][0], T[0][1], T[0][2]],
+            [T[1][0], T[1][1], T[1][2]]
+        ], dtype=np.float64)