Add fecnet

.gitignore

@@ -2,4 +2,8 @@
 .env
 __pycache__
 .vscode
-.cache
+.cache
+.DS_Store
+model/images
+model/data
+model/training_dataset

Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.7
 WORKDIR /code
 
 COPY ./requirements.txt /code/requirements.txt
-RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin  -y
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 RUN useradd -m -u 1000 user

app.py

@@ -1,20 +1,18 @@
-import hmac
 import os
-from typing import Union
 
 import numpy as np
-from fastapi import Depends, FastAPI, File, HTTPException, Response, UploadFile, status
-from fastapi.security import APIKeyQuery
+from fastapi import Depends, FastAPI, File, Response, UploadFile, status
 
-from model import Model
+from auth import validate_token
+from model.model import Model
+from routers.fecnet_router import router as fecnet_router
 from schema import EmbeddingResponse, SimilarityResponse
 
 app = FastAPI(
     title="Facial Expression Embedding Service",
 )
 
-api_key = APIKeyQuery(name="token", auto_error=False)
-client_token: str = os.getenv("CLIENT_TOKEN", "")
+app.include_router(fecnet_router)
 
 model = Model(
     os.getenv("MODEL_REPO_ID", ""),
@@ -23,16 +21,6 @@ model = Model(
 )
 
 
-async def validate_token(
-    token: Union[str, None] = Depends(api_key),
-):
-    if token is None:
-        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "No token provided")
-    if not hmac.compare_digest(token, client_token):
-        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid token")
-    return token
-
-
 @app.post(
     "/embed",
     status_code=status.HTTP_200_OK,

auth.py

@@ -0,0 +1,19 @@
+import hmac
+import os
+from typing import Union
+
+from fastapi import Depends, HTTPException, status
+from fastapi.security import APIKeyQuery
+
+api_key = APIKeyQuery(name="token", auto_error=False)
+client_token: str = os.getenv("CLIENT_TOKEN", "")
+
+
+async def validate_token(
+    token: Union[str, None] = Depends(api_key),
+):
+    if token is None:
+        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "No token provided")
+    if not hmac.compare_digest(token, client_token):
+        raise HTTPException(status.HTTP_401_UNAUTHORIZED, "Invalid token")
+    return token

dev-requirements.txt

@@ -0,0 +1,5 @@
+ipykernel
+plotly
+retrying
+swifter
+nbformat

model/facial_expression_embedding.ipynb

@@ -0,0 +1,808 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Facial Expression Embedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-04-19 12:14:54.399884: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
+      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import os\n",
+    "import random\n",
+    "from typing import *\n",
+    "import tensorflow as tf\n",
+    "from pathlib import Path\n",
+    "from tensorflow.keras import applications\n",
+    "from tensorflow.keras import layers\n",
+    "from tensorflow.keras import losses\n",
+    "from tensorflow.keras import optimizers\n",
+    "from tensorflow.keras import metrics\n",
+    "from tensorflow.keras import Model\n",
+    "from tensorflow.keras.applications import resnet\n",
+    "import pandas as pd\n",
+    "import mediapipe as mp\n",
+    "import plotly.express as px\n",
+    "from plotly.subplots import make_subplots\n",
+    "import plotly.graph_objects as go\n",
+    "import requests\n",
+    "from tqdm import tqdm\n",
+    "import base64\n",
+    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
+    "from retrying import retry\n",
+    "import swifter\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Dataset Loading"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "IMAGE_DIR = \"images\"\n",
+    "TRAINING_DATASET = \"training_dataset\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@retry(stop_max_attempt_number=3)\n",
+    "def image_downloader(url: str):\n",
+    "    get_name = lambda url: base64.urlsafe_b64encode(url.encode()).decode()\n",
+    "    Path(IMAGE_DIR).mkdir(exist_ok=True)\n",
+    "    filename = get_name(url)\n",
+    "    if os.path.exists(os.path.join(IMAGE_DIR, filename)):\n",
+    "        return filename\n",
+    "    res = requests.get(url, timeout=10)\n",
+    "    if not res.ok:\n",
+    "        return None\n",
+    "    with open(os.path.join(IMAGE_DIR, filename), \"wb\") as f:\n",
+    "        f.write(res.content)\n",
+    "    return filename\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_column_names():\n",
+    "    names = []\n",
+    "    for i in range(1, 4):\n",
+    "        names += [\n",
+    "            f\"img{i}_url\",\n",
+    "            f\"img{i}_tl_col\",\n",
+    "            f\"img{i}_br_col\",\n",
+    "            f\"img{i}_tl_row\",\n",
+    "            f\"img{i}_br_row\",\n",
+    "        ]\n",
+    "    names += [\"triplet_type\"]\n",
+    "    for i in range(6):\n",
+    "        names += [f\"annotator{i+1}_id\", f\"annotation{i+1}\"]\n",
+    "    return names\n",
+    "\n",
+    "\n",
+    "def get_local_storage_column_names():\n",
+    "    names = []\n",
+    "    for i in range(1, 4):\n",
+    "        names += [\n",
+    "            f\"img{i}_id\",\n",
+    "            f\"img{i}_tl_col\",\n",
+    "            f\"img{i}_br_col\",\n",
+    "            f\"img{i}_tl_row\",\n",
+    "            f\"img{i}_br_row\",\n",
+    "        ]\n",
+    "    names += [\"triplet_type\"]\n",
+    "    names += [\"annotator1_id\"]\n",
+    "    names += [\"annotation\"]\n",
+    "    return names\n",
+    "\n",
+    "\n",
+    "def get_label(annotations: pd.Series):\n",
+    "    def mode(x):\n",
+    "        s = pd.Series(x)\n",
+    "        if s.value_counts(normalize=True).max() < 0.5:\n",
+    "            return np.nan\n",
+    "        return s.mode().at[0]\n",
+    "\n",
+    "    return annotations.swifter.apply(mode)\n",
+    "\n",
+    "\n",
+    "def fecnet_dataset_loader(dataset_csv: str):\n",
+    "    if isinstance(dataset_csv, bytes):\n",
+    "        dataset_csv = dataset_csv.decode()\n",
+    "    df = pd.read_csv(\n",
+    "        dataset_csv, header=None, names=get_column_names(), nrows=10000\n",
+    "    )  # TODO: remove nrows\n",
+    "\n",
+    "    # download images\n",
+    "    df[\"img1_url\"] = df[\"img1_url\"].swifter.apply(image_downloader)\n",
+    "    df[\"img2_url\"] = df[\"img2_url\"].swifter.apply(image_downloader)\n",
+    "    df[\"img3_url\"] = df[\"img3_url\"].swifter.apply(image_downloader)\n",
+    "    df.dropna(subset=[\"img1_url\", \"img2_url\", \"img3_url\"], inplace=True)\n",
+    "\n",
+    "    # determine label\n",
+    "    df[\"label\"] = get_label(\n",
+    "        pd.Series(df[[f\"annotation{i}\" for i in range(1, 7)]].values.tolist())\n",
+    "    )\n",
+    "    df.dropna(subset=[\"label\"], inplace=True)\n",
+    "\n",
+    "    samples = {\n",
+    "        \"img1\": [],\n",
+    "        \"img1_box\": [],\n",
+    "        \"img2\": [],\n",
+    "        \"img2_box\": [],\n",
+    "        \"img3\": [],\n",
+    "        \"img3_box\": [],\n",
+    "    }\n",
+    "\n",
+    "    for _, row in df.iterrows():\n",
+    "        img1_idx, img2_idx, img3_idx = 1, 2, 3\n",
+    "        if row.label == 1:\n",
+    "            img1_idx, img3_idx = img3_idx, img1_idx\n",
+    "        elif row.label == 2:\n",
+    "            img2_idx, img3_idx = img3_idx, img2_idx\n",
+    "        bounding_boxes = (\n",
+    "            (\n",
+    "                (row[f\"img{img1_idx}_tl_col\"], row[f\"img{img1_idx}_tl_row\"]),\n",
+    "                (row[f\"img{img1_idx}_br_col\"], row[f\"img{img1_idx}_br_row\"]),\n",
+    "            ),\n",
+    "            (\n",
+    "                (row[f\"img{img2_idx}_tl_col\"], row[f\"img{img2_idx}_tl_row\"]),\n",
+    "                (row[f\"img{img2_idx}_br_col\"], row[f\"img{img2_idx}_br_row\"]),\n",
+    "            ),\n",
+    "            (\n",
+    "                (row[f\"img{img3_idx}_tl_col\"], row[f\"img{img3_idx}_tl_row\"]),\n",
+    "                (row[f\"img{img3_idx}_br_col\"], row[f\"img{img3_idx}_br_row\"]),\n",
+    "            ),\n",
+    "        )\n",
+    "        samples[\"img1\"].append(row[f\"img{img1_idx}_url\"])\n",
+    "        samples[\"img1_box\"].append(bounding_boxes[0])\n",
+    "        samples[\"img2\"].append(row[f\"img{img2_idx}_url\"])\n",
+    "        samples[\"img2_box\"].append(bounding_boxes[1])\n",
+    "        samples[\"img3\"].append(row[f\"img{img3_idx}_url\"])\n",
+    "        samples[\"img3_box\"].append(bounding_boxes[2])\n",
+    "    return samples\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_landmarks(image):\n",
+    "    with mp.solutions.face_mesh.FaceMesh(\n",
+    "        static_image_mode=True,\n",
+    "        max_num_faces=1,\n",
+    "        refine_landmarks=True,\n",
+    "        min_detection_confidence=0.5,\n",
+    "    ) as face_mesh:\n",
+    "        results = face_mesh.process(image.numpy())\n",
+    "        if results.multi_face_landmarks:\n",
+    "            landmarks = results.multi_face_landmarks[0]\n",
+    "            landmarks = np.array(\n",
+    "                [[lm.x, lm.y, lm.z] for lm in landmarks.landmark], dtype=np.float32\n",
+    "            )\n",
+    "            landmarks = landmarks.flatten()\n",
+    "        else:\n",
+    "            landmarks = np.zeros(478 * 3, dtype=np.float32)\n",
+    "    return landmarks\n",
+    "\n",
+    "\n",
+    "def preprocess_image(filename: str, tl: Tuple[float, float], br: Tuple[float, float]):\n",
+    "    image_string = tf.io.read_file(tf.strings.join([IMAGE_DIR, \"/\", filename]))\n",
+    "    image = tf.image.decode_jpeg(image_string, channels=3)\n",
+    "    image = tf.image.convert_image_dtype(image, tf.uint8)\n",
+    "\n",
+    "    # crop image\n",
+    "    tl = tf.cast(tf.multiply(tl, tf.cast(tf.shape(image)[:2][::-1], tf.float32)), tf.int32)\n",
+    "    br = tf.cast(tf.multiply(br, tf.cast(tf.shape(image)[:2][::-1], tf.float32)), tf.int32)\n",
+    "    image = tf.image.crop_to_bounding_box(\n",
+    "        image, tl[1], tl[0], br[1] - tl[1], br[0] - tl[0]\n",
+    "    )\n",
+    "\n",
+    "    # extract landmarks using facemesh\n",
+    "    return tf.py_function(extract_landmarks, [image], tf.float32)\n",
+    "    # return image\n",
+    "\n",
+    "\n",
+    "ImgType = Tuple[str, Tuple[float, float], Tuple[float, float]]\n",
+    "\n",
+    "\n",
+    "def preprocess_triplets(triplet: dict):\n",
+    "    anchor: ImgType = (triplet[\"img1\"], triplet[\"img1_box\"][0], triplet[\"img1_box\"][1])\n",
+    "    positive: ImgType = (\n",
+    "        triplet[\"img2\"],\n",
+    "        triplet[\"img2_box\"][0],\n",
+    "        triplet[\"img2_box\"][1],\n",
+    "    )\n",
+    "    negative: ImgType = (\n",
+    "        triplet[\"img3\"],\n",
+    "        triplet[\"img3_box\"][0],\n",
+    "        triplet[\"img3_box\"][1],\n",
+    "    )\n",
+    "    return (\n",
+    "        preprocess_image(*anchor),\n",
+    "        preprocess_image(*positive),\n",
+    "        preprocess_image(*negative),\n",
+    "    )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dc545e1717044afd9ddc0b32fecd72c3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pandas Apply:   0%|          | 0/10000 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0e653babd8104a72a0e4425167b44005",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pandas Apply:   0%|          | 0/10000 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5fe7ddb8d18c41c599a69b9ca87cd8cd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pandas Apply:   0%|          | 0/10000 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "38e2dc2d2517465f896d4fe4ca05da75",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Pandas Apply:   0%|          | 0/8143 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-04-19 13:04:21.919175: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
+      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    }
+   ],
+   "source": [
+    "if not TRAINING_DATASET:\n",
+    "    df = fecnet_dataset_loader(\"data/faceexp-comparison-data-train-public.csv\")\n",
+    "    image_count = len(df)\n",
+    "\n",
+    "    dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "        df\n",
+    "    )\n",
+    "\n",
+    "    dataset = dataset.shuffle(buffer_size=1024)\n",
+    "    dataset = dataset.map(preprocess_triplets)\n",
+    "else:\n",
+    "    dataset = tf.data.Dataset.load(TRAINING_DATASET)\n",
+    "\n",
+    "train_dataset = dataset.take(round(image_count * 0.8))\n",
+    "val_dataset = dataset.skip(round(image_count * 0.8))\n",
+    "\n",
+    "train_dataset = train_dataset.batch(32, drop_remainder=False)\n",
+    "train_dataset = train_dataset.prefetch(8)\n",
+    "\n",
+    "val_dataset = val_dataset.batch(32, drop_remainder=False)\n",
+    "val_dataset = val_dataset.prefetch(8)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Data Visualisation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def visualise_face_mesh(landmarks):\n",
+    "    landmarks = landmarks.reshape(-1, 3)\n",
+    "    fig = go.Figure(\n",
+    "        go.Mesh3d(\n",
+    "            x=landmarks[:, 0],\n",
+    "            y=landmarks[:, 1],\n",
+    "            z=landmarks[:, 2],\n",
+    "            color=\"lightpink\",\n",
+    "            opacity=0.50,\n",
+    "            contour=dict(  \n",
+    "                color=\"grey\",\n",
+    "                width=1,\n",
+    "            ),\n",
+    "        )\n",
+    "    )\n",
+    "    fig.show()\n",
+    "\n",
+    "def visualise_face_mesh_triplets(anchor, positive, negative):\n",
+    "    fig = make_subplots(\n",
+    "        rows=anchor.shape[0],\n",
+    "        cols=3,\n",
+    "        specs=[[{\"type\": \"surface\"}, {\"type\": \"surface\"}, {\"type\": \"surface\"}]],\n",
+    "    )\n",
+    "    for i in range(anchor.shape[0]):\n",
+    "        fig.add_trace(\n",
+    "            go.Mesh3d(\n",
+    "                x=anchor[i, :, 0],\n",
+    "                y=anchor[i, :, 1],\n",
+    "                z=anchor[i, :, 2],\n",
+    "                color=\"lightpink\",\n",
+    "                opacity=0.50,\n",
+    "            ),\n",
+    "            row=i + 1,\n",
+    "            col=1,\n",
+    "        )\n",
+    "        fig.add_trace(\n",
+    "            go.Mesh3d(\n",
+    "                x=positive[i, :, 0],\n",
+    "                y=positive[i, :, 1],\n",
+    "                z=positive[i, :, 2],\n",
+    "                color=\"lightpink\",\n",
+    "                opacity=0.50,\n",
+    "            ),\n",
+    "            row=i + 1,\n",
+    "            col=2,\n",
+    "        )\n",
+    "        fig.add_trace(\n",
+    "            go.Mesh3d(\n",
+    "                x=negative[i, :, 0],\n",
+    "                y=negative[i, :, 1],\n",
+    "                z=negative[i, :, 2],\n",
+    "                color=\"lightpink\",\n",
+    "                opacity=0.50,\n",
+    "            ),\n",
+    "            row=i + 1,\n",
+    "            col=3,\n",
+    "        )\n",
+    "    fig.show()\n",
+    "\n",
+    "def visualise_image(image):\n",
+    "    fig = px.imshow(image)\n",
+    "    fig.show()\n",
+    "\n",
+    "\n",
+    "def visualise_triplet(anchor, positive, negative):\n",
+    "    pass\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 160,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ds_samples = list(dataset.as_numpy_iterator())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 161,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# visualise_face_mesh(ds_samples[30][0])"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Model Definition"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_embedding_model():\n",
+    "    input_layer = layers.Input(shape=(478, 3))\n",
+    "    flatten = layers.Flatten()(input_layer)\n",
+    "    dense1 = layers.Dense(512, activation=\"relu\")(flatten)\n",
+    "    dense1 = layers.BatchNormalization()(dense1)\n",
+    "    dense2 = layers.Dense(256, activation=\"relu\")(dense1)\n",
+    "    dense2 = layers.BatchNormalization()(dense2)\n",
+    "    output = layers.Dense(16)(dense2)\n",
+    "\n",
+    "    embedding = Model(input_layer, output, name=\"Embedding\")\n",
+    "    return embedding"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model: \"Embedding\"\n",
+      "_________________________________________________________________\n",
+      " Layer (type)                Output Shape              Param #   \n",
+      "=================================================================\n",
+      " input_1 (InputLayer)        [(None, 478, 3)]          0         \n",
+      "                                                                 \n",
+      " flatten (Flatten)           (None, 1434)              0         \n",
+      "                                                                 \n",
+      " dense (Dense)               (None, 512)               734720    \n",
+      "                                                                 \n",
+      " batch_normalization (BatchN  (None, 512)              2048      \n",
+      " ormalization)                                                   \n",
+      "                                                                 \n",
+      " dense_1 (Dense)             (None, 256)               131328    \n",
+      "                                                                 \n",
+      " batch_normalization_1 (Batc  (None, 256)              1024      \n",
+      " hNormalization)                                                 \n",
+      "                                                                 \n",
+      " dense_2 (Dense)             (None, 16)                4112      \n",
+      "                                                                 \n",
+      "=================================================================\n",
+      "Total params: 873,232\n",
+      "Trainable params: 871,696\n",
+      "Non-trainable params: 1,536\n",
+      "_________________________________________________________________\n"
+     ]
+    }
+   ],
+   "source": [
+    "embedding = create_embedding_model()\n",
+    "embedding.summary()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class DistanceLayer(layers.Layer):\n",
+    "    def __init__(self, **kwargs):\n",
+    "        super().__init__(**kwargs)\n",
+    "\n",
+    "    def call(self, anchor, positive, negative):\n",
+    "        ap_distance = tf.reduce_sum(tf.square(anchor - positive), -1)\n",
+    "        an_distance = tf.reduce_sum(tf.square(anchor - negative), -1)\n",
+    "        return (ap_distance, an_distance)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "anchor_input = layers.Input(name=\"anchor\", shape=(478,) + (3,))\n",
+    "positive_input = layers.Input(name=\"positive\", shape=(478,) + (3,))\n",
+    "negative_input = layers.Input(name=\"negative\", shape=(478,) + (3,))\n",
+    "\n",
+    "distances = DistanceLayer()(\n",
+    "    embedding(anchor_input),\n",
+    "    embedding(positive_input),\n",
+    "    embedding(negative_input),\n",
+    ")\n",
+    "\n",
+    "siamese_network = Model(\n",
+    "    inputs=[anchor_input, positive_input, negative_input], outputs=distances\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class SiameseModel(Model):\n",
+    "    \"\"\"The Siamese Network model with a custom training and testing loops.\n",
+    "\n",
+    "    Computes the triplet loss using the three embeddings produced by the\n",
+    "    Siamese Network.\n",
+    "\n",
+    "    The triplet loss is defined as:\n",
+    "       L(A, P, N) = max(‖f(A) - f(P)‖² - ‖f(A) - f(N)‖² + margin, 0)\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self, siamese_network, margin=0.2):\n",
+    "        super().__init__()\n",
+    "        self.siamese_network = siamese_network\n",
+    "        self.margin = margin\n",
+    "        self.loss_tracker = metrics.Mean(name=\"loss\")\n",
+    "        self.acc_tracker = metrics.Mean(name=\"accuracy\")\n",
+    "\n",
+    "    def call(self, inputs):\n",
+    "        return self.siamese_network(inputs)\n",
+    "\n",
+    "    def train_step(self, data):\n",
+    "        # GradientTape is a context manager that records every operation that\n",
+    "        # you do inside. We are using it here to compute the loss so we can get\n",
+    "        # the gradients and apply them using the optimizer specified in\n",
+    "        # `compile()`.\n",
+    "        with tf.GradientTape() as tape:\n",
+    "            loss = self._compute_loss(data)\n",
+    "\n",
+    "        # Storing the gradients of the loss function with respect to the\n",
+    "        # weights/parameters.\n",
+    "        gradients = tape.gradient(loss, self.siamese_network.trainable_weights)\n",
+    "\n",
+    "        # Applying the gradients on the model using the specified optimizer\n",
+    "        self.optimizer.apply_gradients(\n",
+    "            zip(gradients, self.siamese_network.trainable_weights)\n",
+    "        )\n",
+    "\n",
+    "        # Let's update and return the training loss metric.\n",
+    "        self.loss_tracker.update_state(loss)\n",
+    "        return {\"loss\": self.loss_tracker.result()}\n",
+    "\n",
+    "    def test_step(self, data):\n",
+    "        loss = self._compute_loss(data)\n",
+    "\n",
+    "        # Let's update and return the loss metric.\n",
+    "        self.loss_tracker.update_state(loss)\n",
+    "        self.acc_tracker.update_state(\n",
+    "            self._compute_accuracy(data)\n",
+    "        )\n",
+    "        return {\"loss\": self.loss_tracker.result(), \"accuracy\": self.acc_tracker.result()}\n",
+    "\n",
+    "    def _compute_loss(self, data):\n",
+    "        # The output of the network is a tuple containing the distances\n",
+    "        # between the anchor and the positive example, and the anchor and\n",
+    "        # the negative example.\n",
+    "        ap_distance, an_distance = self.siamese_network(data)\n",
+    "\n",
+    "        # Computing the Triplet Loss by subtracting both distances and\n",
+    "        # making sure we don't get a negative value.\n",
+    "        loss = ap_distance - an_distance\n",
+    "        loss = tf.maximum(loss + self.margin, 0.0)\n",
+    "        return loss\n",
+    "    \n",
+    "    def _compute_accuracy(self, data):\n",
+    "        ap_distance, an_distance = self.siamese_network(data)\n",
+    "        return tf.cast(ap_distance < an_distance, tf.float32)\n",
+    "\n",
+    "    @property\n",
+    "    def metrics(self):\n",
+    "        # We need to list our metrics here so the `reset_states()` can be\n",
+    "        # called automatically.\n",
+    "        return [self.loss_tracker, self.acc_tracker]"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Model Fitting"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/40\n",
+      "1/1 [==============================] - 1010s 1010s/step - loss: 0.6925 - val_loss: 0.4313 - val_accuracy: 0.5041\n",
+      "Epoch 2/40\n",
+      "1/1 [==============================] - ETA: 0s - loss: 0.3325"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m/var/folders/9f/9272h8p951zfstxdwzn0cj2h0000gn/T/ipykernel_61590/1150799400.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0msiamese_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSiameseModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msiamese_network\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0msiamese_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAdam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.0005\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweighted_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"accuracy\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msiamese_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m40\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1703\u001b[0m                         \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1704\u001b[0m                         \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m                         \u001b[0m_use_cached_eval_dataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1706\u001b[0m                     )\n\u001b[1;32m   1707\u001b[0m                     val_logs = {\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[1;32m   2038\u001b[0m                         ):\n\u001b[1;32m   2039\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2040\u001b[0;31m                             \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2041\u001b[0m                             \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2042\u001b[0m                                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    878\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    882\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0;31m# In this case we have not created variables on the first call. So we can\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    918\u001b[0m       \u001b[0;31m# run the first trace but we should fail if variables are created.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 919\u001b[0;31m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variable_creation_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    920\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_variables\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mALLOW_DYNAMIC_VARIABLE_CREATION\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    921\u001b[0m         raise ValueError(\"Creating variables on a non-first call to a function\"\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    133\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m    134\u001b[0m     return concrete_function._call_flat(\n\u001b[0;32m--> 135\u001b[0;31m         filtered_flat_args, captured_inputs=concrete_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m    136\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    381\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 383\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    384\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    385\u001b[0m           outputs = execute.execute_with_cancellation(\n",
+      "\u001b[0;32m~/Git/FacialExpressionSyncService/.venv/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 53\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     54\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "siamese_model = SiameseModel(siamese_network)\n",
+    "siamese_model.compile(optimizer=optimizers.Adam(0.0005), weighted_metrics=[\"accuracy\"])\n",
+    "siamese_model.fit(train_dataset, epochs=40, validation_data=val_dataset)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Evaluate Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_df = fecnet_dataset_loader(\"data/faceexp-comparison-data-test-public.csv\")\n",
+    "\n",
+    "test_dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "    test_df\n",
+    ")\n",
+    "\n",
+    "test_dataset = test_dataset.map(preprocess_triplets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "siamese_model.evaluate(test_dataset)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 168,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Positive similarity: 0.96082336\n",
+      "Negative similarity 0.85784876\n",
+      "Positive-Negative similarity 0.8386482\n"
+     ]
+    }
+   ],
+   "source": [
+    "sample = next(iter(train_dataset))\n",
+    "# visualise_face_mesh_triplet(*sample)\n",
+    "\n",
+    "anchor, positive, negative = sample\n",
+    "anchor_embedding, positive_embedding, negative_embedding = (\n",
+    "    embedding(tf.reshape(anchor, (-1, 478,3))),\n",
+    "    embedding(tf.reshape(positive, (-1, 478,3))),\n",
+    "    embedding(tf.reshape(negative, (-1, 478,3))),\n",
+    ")\n",
+    "cosine_similarity = metrics.CosineSimilarity()\n",
+    "\n",
+    "positive_similarity = cosine_similarity(anchor_embedding, positive_embedding)\n",
+    "print(\"Positive similarity:\", positive_similarity.numpy())\n",
+    "\n",
+    "negative_similarity = cosine_similarity(anchor_embedding, negative_embedding)\n",
+    "print(\"Negative similarity\", negative_similarity.numpy())\n",
+    "\n",
+    "positive_negative_similarity = cosine_similarity(positive_embedding, negative_embedding)\n",
+    "print(\"Positive-Negative similarity\", positive_negative_similarity.numpy())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO: Created TensorFlow Lite XNNPACK delegate for CPU.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# tf.data.Dataset.save(dataset, \"training_dataset\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.9"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

model/fecnet.py

@@ -0,0 +1,72 @@
+import os
+import sys
+from importlib import import_module, invalidate_caches
+from importlib.util import module_from_spec, spec_from_file_location
+from tempfile import TemporaryDirectory
+
+import cv2
+import numpy as np
+import plotly.express as px
+import requests
+import torch
+from git import Repo
+from huggingface_hub import hf_hub_download
+
+
+class FECNetModel:
+    def __init__(self, hf_token: str) -> None:
+        self.hf_token = hf_token
+        repo_dir = TemporaryDirectory()
+        Repo.clone_from(
+            "https://github.com/AmirSh15/FECNet.git",
+            repo_dir.name,
+        )
+        invalidate_caches()
+        sys.path.append(repo_dir.name)
+        fecnet_module_path = os.path.join(repo_dir.name, "models", "FECNet.py")
+        with open(fecnet_module_path, "r") as f:
+            content = f.read()
+            content = content.replace(
+                "cuda",
+                "cpu",
+            )
+        with open(fecnet_module_path, "w") as f:
+            f.write(content)
+        spec = spec_from_file_location("FECNet", fecnet_module_path)
+        fecnet_module = module_from_spec(spec)  # type: ignore
+        spec.loader.exec_module(fecnet_module)  # type: ignore
+
+        self.model = self.__load_model(
+            self.__download_weights(repo_dir.name), fecnet_module.FECNet
+        )
+
+    def __download_weights(self, model_dir: str) -> str:
+        model_path = hf_hub_download(
+            "natexcvi/pretrained-fecnet",
+            "fecnet.pt",
+            token=self.hf_token,
+        )
+        return model_path
+
+    def __load_model(self, model_path: str, model_class):
+        model = model_class(pretrained=False)
+        model_weights = torch.load(model_path, map_location=torch.device("cpu"))
+        model.load_state_dict(model_weights)
+        model.eval()
+        return model.double()
+
+    def predict(self, image: np.ndarray):
+        pred = self.model.forward(image)
+        return pred
+
+    def distance(a, b):
+        return np.linalg.norm(a - b)
+
+    def embed_image(self, image) -> np.ndarray:
+        image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+        image = cv2.resize(image, (224, 224))
+        image = np.transpose(image, (2, 0, 1))
+        image = np.expand_dims(image, axis=0)
+        image = torch.from_numpy(image).double()
+        pred = self.predict(image)
+        return pred.detach().numpy()

model/fecnet_test.py

@@ -0,0 +1,20 @@
+import os
+
+import numpy as np
+import pytest
+from dotenv import load_dotenv
+from fecnet import FECNetModel
+
+load_dotenv()
+
+
+@pytest.fixture()
+def model():
+    return FECNetModel(hf_token=os.getenv("HF_TOKEN"))
+
+
+def test_embed(model: FECNetModel):
+    image = open("testdata/face_pic.jpeg", "rb").read()
+    image_arr = np.asarray(bytearray(image), dtype=np.uint8)
+    rep = model.embed_image(image_arr)
+    assert rep.shape == (1, 16)

model/openface_model.py

@@ -0,0 +1,27 @@
+import cv2
+import numpy as np
+import openface
+import docker
+
+
+class OpenFaceModel:
+    def __init__(self) -> None:
+        self.client = docker.from_env()
+        self.client.images.pull("bamos/openface")
+
+    def preprocess(self, image: bytes) -> np.ndarray:
+        raise NotImplemented
+
+    def embed(self, aligned_face):
+        container = self.client.containers.run(
+            "bamos/openface",
+            "python /root/openface/demos/classifier.py infer /root/openface/models/openface/celeb-classifier.nn4.small2.v1.pkl -",
+            detach=True,
+            stdin_open=True,
+            tty=True,
+        )
+        raise NotImplemented
+        return rep
+
+    def similarity(self, rep1, rep2):
+        return np.linalg.norm(rep1 - rep2, ord=2)

model/openface_model_test.py

@@ -0,0 +1,14 @@
+import pytest
+from openface_model import OpenFaceModel
+
+
+@pytest.fixture()
+def model():
+    return OpenFaceModel()
+
+@pytest.mark.skip(reason="Not implemented")
+def test_embed(model):
+    image = open("../testdata/face_pic.jpeg", "rb").read()
+    aligned_face = model.preprocess(image)
+    rep = model.embed(aligned_face)
+    assert rep.shape == (128,)

requirements.txt

@@ -9,4 +9,9 @@ mediapipe
 pandas
 pytest
 python-dotenv
-bcrypt
+bcrypt
+openface
+dlib
+docker
+torch
+gitpython

routers/fecnet_router.py

@@ -0,0 +1,33 @@
+from fastapi import APIRouter, Depends, File, UploadFile, status
+
+from auth import validate_token
+from schema import EmbeddingResponse, SimilarityResponse
+
+router = APIRouter(
+    prefix="/fecnet",
+    tags=["fecnet"],
+    dependencies=[Depends(validate_token)],
+)
+
+
+@router.get(
+    "/embed",
+    status_code=status.HTTP_200_OK,
+    response_model=EmbeddingResponse,
+)
+async def calculate_embedding(
+    image: UploadFile = File(...),
+):
+    return {"message": "Hello World"}
+
+
+@router.get(
+    "/similarity",
+    status_code=status.HTTP_200_OK,
+    response_model=SimilarityResponse,
+)
+async def calculate_similarity_score(
+    image1: UploadFile = File(...),
+    image2: UploadFile = File(...),
+):
+    return {"message": "Hello World"}

routers/openface_router.py

@@ -0,0 +1,33 @@
+from fastapi import APIRouter, Depends, File, UploadFile, status
+
+from auth import validate_token
+from schema import EmbeddingResponse, SimilarityResponse
+
+router = APIRouter(
+    prefix="/openface",
+    tags=["openface"],
+    dependencies=[Depends(validate_token)],
+)
+
+
+@router.get(
+    "/embed",
+    status_code=status.HTTP_200_OK,
+    response_model=EmbeddingResponse,
+)
+async def calculate_embedding(
+    image: UploadFile = File(...),
+):
+    return {"message": "Hello World"}
+
+
+@router.get(
+    "/similarity",
+    status_code=status.HTTP_200_OK,
+    response_model=SimilarityResponse,
+)
+async def calculate_similarity_score(
+    image1: UploadFile = File(...),
+    image2: UploadFile = File(...),
+):
+    return {"message": "Hello World"}