First working version of app

.gitattributes

@@ -1 +1,2 @@
+weights filter=lfs diff=lfs merge=lfs -text
 weights/*.pt filter=lfs diff=lfs merge=lfs -text

environment.yml

@@ -5,7 +5,7 @@ dependencies:
   - python
   - ipykernel
   - pip
-  - ultralytics
+  - ultralytics=8.0.186
   - pillow
   - python-dotenv
   - opencv

notebooks/app.ipynb

@@ -2,135 +2,125 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 179,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import ipywidgets as widgets"
+    "from ultralytics import YOLO, RTDETR\n",
+    "from pathlib import Path \n",
+    "import cv2\n",
+    "from dotenv import load_dotenv\n",
+    "import os\n",
+    "import requests\n",
+    "import urllib\n",
+    "from io import BytesIO\n",
+    "from PIL import Image, ImageDraw\n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import ipywidgets as widgets\n",
+    "\n",
+    "%matplotlib inline\n",
+    "\n",
+    "os.chdir('/home/camille/code/surveillance/')\n",
+    "_ = load_dotenv()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Re-surveilling surveillance\n",
+    "\n",
+    "This is a demonstration of using object detection models to locate surveillance cameras in Google Street View imagery. There are two models: **YOLO**, which runs relatively quickly but is less accurate, and **RT-DETR**, a newer model type that is slower but better at finding cameras.\n",
+    "\n",
+    "Enter a location to look up and adjust the camera parameters. After clicking the button, any cameras detected by the models will be highlighted."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 180,
    "metadata": {},
    "outputs": [],
    "source": [
+    "yolo = YOLO('./weights/yolo_tile_best.pt')\n",
+    "detr = RTDETR('./weights/detr_tile_frz_best.pt')\n",
+    "\n",
     "DEFAULTS = {\n",
     "    'size': 640,\n",
-    "    'heading': 140,\n",
+    "    'heading': 160,\n",
     "    'pitch': 10,\n",
-    "    'fov': 50\n",
-    "}"
+    "    'fov': 50,\n",
+    "    'model': 'detr'\n",
+    "}\n",
+    "\n",
+    "rows = 4\n",
+    "cols = 3\n",
+    "grid = widgets.GridspecLayout(rows, cols, \n",
+    "                              min_height = '400px', \n",
+    "                              grid_gap = '8px',\n",
+    "                              merge = False)\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 181,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_sv_img(\n",
+    "    location: str,  \n",
+    "    key_name: str = 'GOOGLE_KEY',\n",
+    "    size: int = DEFAULTS['size'],\n",
+    "    heading: int = DEFAULTS['heading'],\n",
+    "    pitch: int = DEFAULTS['pitch'],\n",
+    "    fov: int = DEFAULTS['fov']\n",
+    ") -> Image:\n",
+    "    img_params = {\n",
+    "        'location': location,\n",
+    "        'size': f'{size}x{size}',\n",
+    "        'heading': heading,\n",
+    "        'pitch': pitch,\n",
+    "        'fov': fov,\n",
+    "        'key': os.getenv(key_name)\n",
+    "    }\n",
+    "    r = requests.get('https://maps.googleapis.com/maps/api/streetview', params = urllib.parse.urlencode(img_params))\n",
+    "    img = Image.open(BytesIO(r.content))\n",
+    "    return img\n",
+    "\n",
+    "def make_img_widget(img: Image, label: str, size: int = 480) -> widgets.Image:\n",
+    "    buff = BytesIO()\n",
+    "    img.save(buff, format = 'PNG')\n",
+    "    img_widget = widgets.Image(value = buff.getvalue(), width = size, height = size)\n",
+    "    grd = widgets.VBox([widgets.Label(label), img_widget])\n",
+    "    return grd\n",
+    "\n",
+    "def label_img(img, model) -> Image:\n",
+    "    pred = model.predict(img, device = 'cpu')[0].plot(labels = False)\n",
+    "    pred = cv2.cvtColor(pred, cv2.COLOR_BGR2RGB)\n",
+    "    return Image.fromarray(pred)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 182,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "57fe1b257f5448208a6e6c07ada26ef8",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Text(value='33rd & Loch Raven Baltimore MD', description='Location', layout=Layout(width='600px'), style=TextS…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6a4f58d9f343425993aa59f5e3271118",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "IntSlider(value=640, description='Image size', layout=Layout(width='600px'), max=1024, min=100, style=SliderSt…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "36da56e191be4d7fb5c26a3e90145d4b",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "IntSlider(value=140, description='Heading (rotation)', layout=Layout(width='600px'), max=360, style=SliderStyl…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cb10a0a70dc04d2690dc41d7eec0bb1f",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "IntSlider(value=10, description='Pitch (tilt)', layout=Layout(width='600px'), max=40, style=SliderStyle(descri…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2425ac1c49504ec199bdee5abee9d2e3",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "IntSlider(value=50, description='Field of view (zoom)', layout=Layout(width='600px'), max=120, min=10, style=S…"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "da832f9cbde14700a60901e887b1789c",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Button(description='Get image', style=ButtonStyle())"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3d8bec926ef0413db6d9865418dc2045",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Output()"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 1 surveillance, 262.8ms\n",
+      "Speed: 1.9ms preprocess, 262.8ms inference, 1.1ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x640 3 surveillances, 988.3ms\n",
+      "Speed: 1.8ms preprocess, 988.3ms inference, 0.6ms postprocess per image at shape (1, 3, 640, 640)\n"
+     ]
     }
    ],
    "source": [
-    "output = widgets.Output()\n",
     "lbl_style = {'description_width': '200px'}\n",
-    "layout = widgets.Layout(width='600px')\n",
+    "layout = widgets.Layout(width='60%')\n",
     "location = widgets.Text(value='33rd & Loch Raven Baltimore MD',\n",
     "                        description='Location',\n",
     "                        layout=layout,\n",
@@ -160,46 +150,63 @@
     "                        style=lbl_style,\n",
     "                        description='Field of view (zoom)')\n",
     "\n",
-    "button = widgets.Button(description='Get image')\n",
+    "button = widgets.Button(description='Get image', button_style = 'primary')\n",
+    "\n",
+    "mod_choice = widgets.Dropdown(value=DEFAULTS['model'],\n",
+    "                              options=[('RT-DETR', 'detr'), ('YOLOv8', 'yolo')],\n",
+    "                              layout=layout,\n",
+    "                              style=lbl_style,\n",
+    "                              description='Type of model')\n",
     "\n",
-    "display(location, size, heading, pitch, fov, button, output)\n",
+    "grid[0, :] = widgets.VBox([location, size, heading, pitch, fov, button],\n",
+    "                          layout = widgets.Layout(height = 'auto'))\n",
     "\n",
     "\n",
     "def button_click(b):\n",
-    "    output.clear_output()\n",
-    "    with output:\n",
-    "        txt = f'location: {location.value}'\n",
-    "        print(txt)\n",
-    "    return txt\n",
+    "    img = get_sv_img(location=location.value,\n",
+    "                     size=size.value,\n",
+    "                     heading=heading.value,\n",
+    "                     pitch=pitch.value,\n",
+    "                     fov=fov.value)\n",
+    "    if img is not None:\n",
+    "        grid[1:, 0] = make_img_widget(img, 'Original image')\n",
+    "        yolo_pred = label_img(img, yolo)\n",
+    "        grid[1:, 1] = make_img_widget(yolo_pred, 'YOLO predictions')\n",
+    "        detr_pred = label_img(img, detr)\n",
+    "        grid[1:, 2] = make_img_widget(detr_pred, 'RT-DETR predictions')\n",
+    "        \n",
     "\n",
-    "sv_img = button.on_click(button_click)\n"
+    "button.on_click(button_click)\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 184,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9020055a57294d38b6f3e40659b12a1d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "GridspecLayout(children=(VBox(children=(Text(value='33rd & Loch Raven Baltimore MD', description='Location', l…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "display(grid)"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "voila",
+   "display_name": "cap",
    "language": "python",
    "name": "python3"
   },

weights/detr_full_frz_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8646e7c3f92c0e3bc96343cf1fad22bb252ccefb1b4cf44a69abbbadd6545b9d
+size 52028929

weights/detr_tile_frz_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7acae2ecb5d1de1bcdf9b51afb305d696a086df4f85b6ecc533901f060eb9683
+size 66137280

weights/yolo_tile_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8646e7c3f92c0e3bc96343cf1fad22bb252ccefb1b4cf44a69abbbadd6545b9d
+size 52028929