Add files

.gitignore

@@ -0,0 +1 @@
+images

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "face_detection"]
+	path = face_detection
+	url = https://github.com/ibug-group/face_detection
+[submodule "deep-head-pose"]
+	path = deep-head-pose
+	url = https://github.com/natanielruiz/deep-head-pose

app.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import argparse
+import functools
+import os
+import pathlib
+import sys
+import tarfile
+from typing import Callable
+
+import cv2
+import gradio as gr
+import huggingface_hub
+import numpy as np
+import PIL.Image
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision
+import torchvision.transforms as T
+from scipy.spatial.transform import Rotation
+
+sys.path.insert(0, 'face_detection')
+sys.path.insert(0, 'deep-head-pose/code')
+
+from hopenet import Hopenet
+from ibug.face_detection import RetinaFacePredictor
+
+TITLE = 'natanielruiz/deep-head-pose'
+DESCRIPTION = 'This is a demo for https://github.com/natanielruiz/deep-head-pose.'
+ARTICLE = None
+
+TOKEN = os.environ['TOKEN']
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--device', type=str, default='cpu')
+    parser.add_argument('--theme', type=str)
+    parser.add_argument('--live', action='store_true')
+    parser.add_argument('--share', action='store_true')
+    parser.add_argument('--port', type=int)
+    parser.add_argument('--disable-queue',
+                        dest='enable_queue',
+                        action='store_false')
+    parser.add_argument('--allow-flagging', type=str, default='never')
+    parser.add_argument('--allow-screenshot', action='store_true')
+    return parser.parse_args()
+
+
+def load_sample_images() -> list[pathlib.Path]:
+    image_dir = pathlib.Path('images')
+    if not image_dir.exists():
+        image_dir.mkdir()
+        dataset_repo = 'hysts/input-images'
+        filenames = ['001.tar']
+        for name in filenames:
+            path = huggingface_hub.hf_hub_download(dataset_repo,
+                                                   name,
+                                                   repo_type='dataset',
+                                                   use_auth_token=TOKEN)
+            with tarfile.open(path) as f:
+                f.extractall(image_dir.as_posix())
+    return sorted(image_dir.rglob('*.jpg'))
+
+
+def load_model(model_name: str, device: torch.device) -> nn.Module:
+    path = huggingface_hub.hf_hub_download('hysts/Hopenet',
+                                           f'models/{model_name}.pkl',
+                                           use_auth_token=TOKEN)
+    state_dict = torch.load(path, map_location='cpu')
+    model = Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66)
+    model.load_state_dict(state_dict)
+    model.to(device)
+    model.eval()
+    return model
+
+
+def create_transform() -> Callable:
+    transform = T.Compose([
+        T.Resize(224),
+        T.CenterCrop(224),
+        T.ToTensor(),
+        T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    return transform
+
+
+def crop_face(image: np.ndarray, box: tuple[int, int, int, int]) -> np.ndarray:
+    x0, y0, x1, y1 = box
+    w = x1 - x0
+    h = y1 - y0
+    x0 -= 2 * w // 4
+    x1 += 2 * w // 4
+    y0 -= 3 * h // 4
+    y1 += h // 4
+    x0 = max(x0, 0)
+    y0 = max(y0, 0)
+    x1 = min(x1, image.shape[1])
+    y1 = min(y1, image.shape[0])
+    image = image[y0:y1, x0:x1]
+    return image
+
+
+@torch.inference_mode()
+def predict(image: np.ndarray, transform: Callable, model: nn.Module,
+            device: torch.device) -> np.ndarray:
+    indices = torch.arange(66).float().to(device)
+
+    image = PIL.Image.fromarray(image)
+    data = transform(image)
+    data = data.to(device)
+
+    # the output of the model is a tuple of 3 tensors (yaw, pitch, roll)
+    # the shape of each tensor is (1, 66)
+    out = model(data[None, ...])
+    out = torch.stack(out, dim=1)  # shape: (1, 3, 66)
+    out = F.softmax(out, dim=2)
+    out = (out * indices).sum(dim=2) * 3 - 99
+    out = out.cpu().numpy()[0]
+    return out
+
+
+def draw_axis(image: np.ndarray, pose: np.ndarray, origin: np.ndarray,
+              length: int) -> None:
+    # (yaw, pitch, roll) -> (roll, yaw, pitch)
+    pose = pose[[2, 0, 1]]
+    pose *= np.array([1, -1, 1])
+    rot = Rotation.from_euler('zyx', pose, degrees=True)
+
+    vectors = rot.as_matrix().T[:, :2]  # shape: (3, 2)
+    pts = np.round(vectors * length + origin).astype(int)
+
+    cv2.line(image, tuple(origin), tuple(pts[0]), (0, 0, 255), 3)
+    cv2.line(image, tuple(origin), tuple(pts[1]), (0, 255, 0), 3)
+    cv2.line(image, tuple(origin), tuple(pts[2]), (255, 0, 0), 2)
+
+
+def run(image: np.ndarray, model_name: str, face_detector: RetinaFacePredictor,
+        models: dict[str, nn.Module], transform: Callable,
+        device: torch.device) -> np.ndarray:
+    model = models[model_name]
+
+    # RGB -> BGR
+    det_faces = face_detector(image[:, :, ::-1], rgb=False)
+
+    res = image[:, :, ::-1].copy()
+    for det_face in det_faces:
+        box = np.round(det_face[:4]).astype(int)
+
+        # RGB
+        face_image = crop_face(image, box.tolist())
+
+        # (yaw, pitch, roll)
+        angles = predict(face_image, transform, model, device)
+
+        center = (box[:2] + box[2:]) // 2
+        length = (box[3] - box[1]) // 2
+        draw_axis(res, angles, center, length)
+
+    return res[:, :, ::-1]
+
+
+def main():
+    gr.close_all()
+
+    args = parse_args()
+    device = torch.device(args.device)
+
+    face_detector = RetinaFacePredictor(
+        threshold=0.8,
+        device=device,
+        model=RetinaFacePredictor.get_model('mobilenet0.25'))
+
+    model_names = [
+        'hopenet_alpha1',
+        'hopenet_alpha2',
+        'hopenet_robust_alpha1',
+    ]
+    models = {name: load_model(name, device) for name in model_names}
+
+    transform = create_transform()
+
+    func = functools.partial(run,
+                             face_detector=face_detector,
+                             models=models,
+                             transform=transform,
+                             device=device)
+    func = functools.update_wrapper(func, run)
+
+    image_paths = load_sample_images()
+    examples = [[path.as_posix(), model_names[0]] for path in image_paths]
+
+    gr.Interface(
+        func,
+        [
+            gr.inputs.Image(type='numpy', label='Input'),
+            gr.inputs.Radio(model_names,
+                            type='value',
+                            default=model_names[0],
+                            label='Model'),
+        ],
+        gr.outputs.Image(type='numpy', label='Output'),
+        examples=examples,
+        title=TITLE,
+        description=DESCRIPTION,
+        article=ARTICLE,
+        theme=args.theme,
+        allow_screenshot=args.allow_screenshot,
+        allow_flagging=args.allow_flagging,
+        live=args.live,
+    ).launch(
+        enable_queue=args.enable_queue,
+        server_port=args.port,
+        share=args.share,
+    )
+
+
+if __name__ == '__main__':
+    main()

deep-head-pose

@@ -0,0 +1 @@
+Subproject commit f7bbb9981c2953c2eca67748d6492a64c8243946

face_detection

@@ -0,0 +1 @@
+Subproject commit bc1e392b11d731fa20b1397c8ff3faed5e7fc76e

requirements.txt

@@ -0,0 +1,6 @@
+numpy==1.22.3
+opencv-python-headless==4.5.5.64
+Pillow==9.1.0
+scipy==1.8.0
+torch==1.11.0
+torchvision==0.12.0