added js

instance-labeler

@@ -1 +0,0 @@
-Subproject commit f4dde8b7122daba0a4cf869dc78d450fe80ffe81

instance-labeler/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

instance-labeler/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

instance-labeler/README.md

@@ -0,0 +1,34 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

instance-labeler/app/canvas.tsx

@@ -0,0 +1,362 @@
+'use client';
+import { useState, useRef, useEffect } from 'react';
+import axios from 'axios';
+import useImage from 'use-image';
+import Konva from 'konva';
+import { Stage, Image, Layer } from 'react-konva';
+import { BASE_URL } from './constants';
+import Dropdown from './dropdown';
+
+
+interface RGB {
+  r: number;
+  g: number;
+  b: number;
+}
+
+interface Box {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+const maskFilter = (imageData: ImageData, color: RGB) => {
+  const nPixels = imageData.data.length;
+  for (let i = 0; i < nPixels; i += 4) {
+    const r = imageData.data[i];
+    const g = imageData.data[i + 1];
+    const b = imageData.data[i + 2];
+
+    if (r === 0 && g === 0 && b === 0) {
+      imageData.data[i + 3] = 0;
+    } else {
+      imageData.data[i] = color.r;
+      imageData.data[i + 1] = color.g;
+      imageData.data[i + 2] = color.b;
+      imageData.data[i + 3] = 128;
+    }
+  }
+};
+
+export default function Canvas({ imageUrl, imageName }: { imageUrl: string, imageName: string }) {
+  const [image] = useImage(imageUrl, 'anonymous');
+  const stageRef = useRef<Konva.Stage>(null);
+  const layerRef = useRef<Konva.Layer>(null);
+  const groupRef = useRef<Konva.Group[]>([]);
+
+  const pointsRef = useRef<Array<[number, number]>>([]);
+  const labelsRef = useRef<Array<number>>([]);
+
+  const instanceColorsRef = useRef<string[]>([]);
+  const [classList, setClassList] = useState<string[]>([]);
+  const classOptionsRef = useRef<string[]>([]);
+  const classSelectionRef = useRef<string>('');
+
+  const [useLatest, setUseLatest] = useState<boolean>(false);
+  const selectedInstanceRef = useRef<number>(-1);
+
+  const [selectedInstanceStyle, setSelectedInstanceStyle] = useState<boolean>(false);
+
+  const clearLayers = () => {
+    groupRef.current.forEach((group) => {
+      group.destroy();
+    });
+    groupRef.current = [];
+    layerRef.current?.draw();
+    pointsRef.current = [];
+    labelsRef.current = [];
+    instanceColorsRef.current = [Konva.Util.getRandomColor()];
+    setClassList(_ => ([]));
+  }
+
+  useEffect(() => {
+    clearLayers();
+  }, [imageUrl]);
+
+  const showMask = (data: string, group: Konva.Group) => {
+    const layer = layerRef.current;
+    if (!layer) return;
+
+    const color = Konva.Util.getRGB(
+      instanceColorsRef.current[instanceColorsRef.current.length - 1]
+    );
+
+    const width = image?.width;
+    const height = image?.height;
+
+    const maskObj = new window.Image();
+    maskObj.onload = () => {
+      const image = new Konva.Image({
+        x: 0,
+        y: 0,
+        image: maskObj,
+        width: width,
+        height: height,
+      });
+      image.filters([(imageData: ImageData) => maskFilter(imageData, color)]);
+      image.cache();
+
+      group.add(image);
+
+      if (groupRef.current.length === 0)
+        groupRef.current.push(group);
+
+      layer.add(group);
+      layer.draw()
+    };
+    maskObj.src = `data:image/png;base64,${data}`;
+    setUseLatest(true);
+  }
+
+  const showBox = (box: Box, group: Konva.Group) => {
+    const layer = layerRef.current;
+    if (!layer) return;
+
+    const color = Konva.Util.getRGB(
+      instanceColorsRef.current[instanceColorsRef.current.length - 1]
+    );
+
+    const width = image?.width;
+    const height = image?.height;
+
+    function rgbToHex(r, g, b) {
+      return '#' + (1 << 24 | r << 16 | g << 8 | b).toString(16).slice(1);
+    }
+
+    const rect = new Konva.Rect({
+      x: box.x,
+      y: box.y,
+      width: box.width,
+      height: box.height,
+      stroke: rgbToHex(color.r, color.g, color.b),
+      storkeWidth: 2,
+    });
+
+    if (groupRef.current.length === 0)
+      groupRef.current.push(group);
+    group.add(rect);
+    layer.add(group);
+    layer.draw();
+    setUseLatest(true);
+  }
+
+  const handleStageClick = async (e: Konva.KonvaEventObject<MouseEvent>) => {
+    const stage = stageRef.current;
+    const layer = layerRef.current;
+    if (!stage || !layer) return;
+
+    const pos = e.target.getStage()?.getPointerPosition();
+    if (!pos) return;
+
+    let labels = labelsRef.current;
+    if (e.evt.button === 2) {
+      labels = [...labels, 0];
+    } else {
+      labels = [...labels, 1];
+    }
+    labelsRef.current = labels;
+
+    // re-draw last layer so we don't overlap masks on clicks
+    if (groupRef.current.length > 0) {
+      groupRef.current[groupRef.current.length - 1].destroy();
+    } else {
+      groupRef.current.push(new Konva.Group());
+    }
+
+    layer.draw();
+
+    let points = pointsRef.current;
+    points = [...points, [pos.x, pos.y]];
+    pointsRef.current = points;
+
+    const res = await axios.post(
+      `${BASE_URL}/v1/get_label_preds/${imageName}`,
+      {
+        points: points,
+        labels: labels,
+      }
+    );
+    showMask(res.data.masks[0], groupRef.current[groupRef.current.length - 1]);
+  }
+
+  const pinInstance = async () => {
+    const layer = layerRef.current;
+    if (!layer) return;
+
+    const groupList = groupRef.current;
+    // if (groupList.length === 0) return;
+
+    if (classSelectionRef.current === '') {
+      alert('Please select a class');
+    } else {
+      groupList.push(new Konva.Group());
+      pointsRef.current = [];
+      labelsRef.current = [];
+      instanceColorsRef.current.push(Konva.Util.getRandomColor());
+      setClassList(prev => ([...prev, classSelectionRef.current]))
+      setUseLatest(false);
+    }
+  }
+
+  const deleteSelectedInstance = () => {
+    const layer = layerRef.current;
+    if (!layer) return;
+    if (selectedInstanceRef.current === -1) return;
+
+    // always 1 extra group for the next instance
+    if (groupRef.current.length <= 1) return;
+
+    groupRef.current[selectedInstanceRef.current].destroy();
+    groupRef.current.splice(selectedInstanceRef.current, 1);
+    setClassList([
+      ...classList.slice(0, selectedInstanceRef.current),
+      ...classList.slice(selectedInstanceRef.current + 1)
+    ]);
+    instanceColorsRef.current.splice(selectedInstanceRef.current, 1);
+    layer.draw()
+    // reset selectedInstanceRef?
+    selectedInstanceRef.current = -1;
+  }
+
+  const submitLabels = async () => {
+    const groupList = groupRef.current;
+    if (groupList.length === 0) return;
+
+    const masks = [];
+    const length = useLatest ? groupList.length : groupList.length - 1;
+    for (let i = 0; i < length; i++) {
+      const labelData = groupList[i].toDataURL({ x: 0, y: 0, width: image?.width, height: image?.height });
+      masks.push(labelData);
+    }
+
+    const res = await axios.put(`${BASE_URL}/v1/label_image/${imageName}`,
+      {
+        masks: masks,
+        labels: classList,
+      }
+    );
+
+    pointsRef.current = [];
+    labelsRef.current = [];
+    clearLayers();
+  }
+
+  const getLabels = async () => {
+    clearLayers();
+    try {
+      const res = await axios.get(`${BASE_URL}/v1/get_labels/${imageName}`);
+      classOptionsRef.current = [...new Set(res.data.labels)];
+      // add an initial group to start with
+      groupRef.current.push(new Konva.Group());
+      for (let i = 0; i < res.data.masks.length; i++) {
+        showMask(res.data.masks[i], groupRef.current[groupRef.current.length - 1]);
+        classSelectionRef.current = res.data.labels[i];
+        pinInstance();
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  const predLabels = async () => {
+    const length = useLatest ? groupRef.current.length : groupRef.current.length - 1;
+    if (groupRef.current.length === 0) {
+      alert('Please pin an instance');
+      return
+    }
+    const mask = groupRef.current[length - 1].toDataURL({ x: 0, y: 0, width: image?.width, height: image?.height });
+
+    clearLayers();
+    try {
+      const res = await axios.post(`${BASE_URL}/v1/get_multi_label_preds/${imageName}`, {
+        mask: mask,
+        label: classList[length - 1],
+      });
+      groupRef.current.push(new Konva.Group());
+      for (let i = 0; i < res.data.masks.length; i++) {
+        showMask(res.data.masks[i], groupRef.current[groupRef.current.length - 1]);
+        classSelectionRef.current = res.data.labels[i];
+        pinInstance();
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  const getBboxes = async () => {
+    clearLayers();
+    try {
+      const res = await axios.get(`${BASE_URL}/v1/get_labels/${imageName}`);
+      classOptionsRef.current = [...new Set(res.data.labels)];
+      groupRef.current.push(new Konva.Group());
+      for (let i = 0; i < res.data.bboxes.length; i++) {
+        showBox(res.data.bboxes[i], groupRef.current[groupRef.current.length - 1]);
+        classSelectionRef.current = res.data.labels[i];
+        pinInstance();
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  return (
+    <div className="flex flex-row columns-2 gap-20">
+      <div className="">
+        <h1>Instance Class List</h1>
+        <ul>
+          {classList.map((item, index) => (
+            <li key={index}>
+              <div
+                // style={{ background: instanceColorsRef.current[index], border: '2px solid red'}}
+                style={{ 
+                  background: instanceColorsRef.current[index], 
+                  border: selectedInstanceRef.current === index && selectedInstanceStyle ? '2px solid red' : 'none'
+                }}
+                onClick={() => {
+                  selectedInstanceRef.current = index;
+                  setSelectedInstanceStyle(prev => !prev)
+                }}
+              >{item}</div>
+            </li>
+          ))}
+        </ul>
+        <br />
+        <Dropdown
+          options={classOptionsRef}
+          selectedOption={classSelectionRef}
+        />
+      </div>
+      <div className="">
+        <Stage
+          width={image?.width}
+          height={image?.height}
+          ref={stageRef}
+          onClick={handleStageClick}
+          onContextMenu={(e) => e.evt.preventDefault()}
+        >
+          <Layer ref={layerRef}>
+            <Image image={image} alt="image" />
+          </Layer>
+        </Stage>
+        <br />
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={clearLayers}>Clear All</button>&nbsp;
+
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={getBboxes}>Load BBoxes</button>&nbsp;
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={getLabels}>Load Masks</button>&nbsp;
+
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={deleteSelectedInstance}>Delete Selected Instance</button>&nbsp;
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={pinInstance}>Pin Instance</button>&nbsp;
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={submitLabels}>Save All</button>&nbsp;
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={predLabels}>Auto Label</button>
+      </div>
+    </div>
+  );
+};

instance-labeler/app/constants.tsx

@@ -0,0 +1 @@
+export const BASE_URL = 'http://localhost:7860'

instance-labeler/app/dropdown.tsx

@@ -0,0 +1,55 @@
+import React, { useState } from 'react';
+
+interface DropdownProps {
+  options: React.RefObject<string[]>;
+  selectedOption: React.RefObject<string>;
+}
+
+
+export default function Dropdown({ options, selectedOption }: DropdownProps) {
+  const [value, setValue] = useState<string>('');
+  const [newValue, setNewValue] = useState<string>('');
+
+  const handleAddItem = () => {
+    options.current?.push(newValue);
+    selectedOption.current = newValue;
+    setNewValue('');
+  };
+
+  const handleSelectItem = (event: React.ChangeEvent<HTMLSelectElement>) => {
+    selectedOption.current = event.target.value;
+    setValue(event.target.value);
+  };
+
+  return (
+    <div>
+      <select 
+        value={value}
+        onChange={(event) => handleSelectItem(event)}
+        style={{ color: 'black' }}
+      >
+        {options.current?.map((option: string, index: number) => (
+          <option 
+            value={option}
+            key={index}
+            style={{ color: 'black' }}
+          >
+            {option}
+          </option>
+        ))}
+      </select>
+      <br />
+      <br />
+      <input
+        type="text"
+        style={{ color: 'black' }}
+        value={newValue}
+        onChange={(e) => setNewValue(e.target.value)}
+      />
+      <br />
+      <br />
+      <button className="bg-gray-400 hover:bg-blue-700 text-white font-bold py-1 px-1"
+        onClick={handleAddItem}>Add</button>&nbsp;
+    </div>
+  );
+};

instance-labeler/app/globals.css

@@ -0,0 +1,27 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --foreground-rgb: 0, 0, 0;
+  --background-start-rgb: 214, 219, 220;
+  --background-end-rgb: 255, 255, 255;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --foreground-rgb: 255, 255, 255;
+    --background-start-rgb: 0, 0, 0;
+    --background-end-rgb: 0, 0, 0;
+  }
+}
+
+body {
+  color: rgb(var(--foreground-rgb));
+  background: linear-gradient(
+      to bottom,
+      transparent,
+      rgb(var(--background-end-rgb))
+    )
+    rgb(var(--background-start-rgb));
+}

instance-labeler/app/layout.tsx

@@ -0,0 +1,21 @@
+import './globals.css'
+import { Inter } from 'next/font/google'
+
+const inter = Inter({ subsets: ['latin'] })
+
+export const metadata = {
+  title: 'Create Next App',
+  description: 'Generated by create next app',
+}
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode
+}) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  )
+}

instance-labeler/app/page.tsx

@@ -0,0 +1,72 @@
+'use client';
+import { useState, useEffect, useRef, ChangeEvent } from 'react';
+import axios from 'axios';
+import Canvas from './canvas';
+import { BASE_URL } from './constants';
+
+
+export default function Home() {
+  const [images, setImages] = useState<string[]>([]);
+  const [displayImage, setDisplayImage] = useState<string>('/loading.png');
+  const [displayIndex, setDisplayIndex] = useState<number>(0);
+
+  const getAllImages = async () => {
+    const res = await axios.get(`${BASE_URL}/v1/get_all_images`);
+    setImages(res.data.images);
+  }
+
+  const setCurrentImage = async () => {
+    if (images[displayIndex] === undefined) return;
+    const res = await axios.get(`${BASE_URL}/v1/get_image/${images[displayIndex]}`);
+    const contentType = res.headers['content-type'];
+    setDisplayImage(`data:${contentType};base64,${res.data}`);
+  }
+
+  const handleFileUpload = async (e: ChangeEvent<HTMLInputElement>) => {
+    const reader = new FileReader();
+    reader.addEventListener("load", async () => {
+      const newImageName = `img${images.length + 1}`;
+      const res = await axios.put(`${BASE_URL}/v1/upload_image/${newImageName}`,
+        {
+          image: reader.result,
+        }
+      );
+      setImages([...images, newImageName]);
+      setDisplayIndex(images.length);
+    }, false);
+
+    if (e.target.files) {
+      for (let i = 0; i < e.target.files.length; i++) {
+        reader.readAsDataURL(e.target.files[i]);
+      }
+    }
+  }
+
+  useEffect(() => {
+    getAllImages();
+  }, [])
+
+  useEffect(() => {
+    setCurrentImage();
+  }, [images, displayIndex])
+
+  return (
+    <>
+      <Canvas imageUrl={displayImage} imageName={images[displayIndex]} />
+      <br />
+      <br />
+      <div className="float right">
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={() => {
+            setDisplayIndex(displayIndex - 1 >= 0 ? displayIndex - 1 : images.length - 1);
+          }}>Previous</button>&nbsp;
+        <button className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-2"
+          onClick={() => {
+            setDisplayIndex(displayIndex + 1 < images.length ? displayIndex + 1 : 0);
+          }}>Next</button>
+      </div>
+      <br />
+      <input type="file" id="image" accept="image/png, image/jpeg" onChange={handleFileUpload} />
+    </>
+  )
+};

instance-labeler/next.config.js

@@ -0,0 +1,19 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  output: 'export',
+  basePath: '',
+  reactStrictMode: true,
+  experimental: {
+    appData: true,
+    esmExternals: "loose", // required to make Konva and react-konva work
+  },
+  typescript: {
+    ignoreBuildErrors: true,
+  },
+  webpack: (config) => {
+    config.externals = [...config.externals, { canvas: "canvas" }]; // required to make Konva and react-konva work
+    return config;
+  },
+};
+
+module.exports = nextConfig

instance-labeler/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "instance-labeler",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@types/node": "20.2.5",
+    "@types/react": "18.2.9",
+    "@types/react-dom": "18.2.4",
+    "autoprefixer": "10.4.14",
+    "axios": "^1.4.0",
+    "canvas": "^2.11.2",
+    "eslint": "8.42.0",
+    "eslint-config-next": "13.4.4",
+    "konva": "^9.2.0",
+    "next": "13.4.4",
+    "postcss": "8.4.24",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-konva": "^18.2.9",
+    "tailwindcss": "3.3.2",
+    "typescript": "5.1.3",
+    "use-image": "^1.1.0"
+  }
+}

instance-labeler/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+}

instance-labeler/tailwind.config.js

@@ -0,0 +1,18 @@
+/** @type {import('tailwindcss').Config} */
+module.exports = {
+  content: [
+    './pages/**/*.{js,ts,jsx,tsx,mdx}',
+    './components/**/*.{js,ts,jsx,tsx,mdx}',
+    './app/**/*.{js,ts,jsx,tsx,mdx}',
+  ],
+  theme: {
+    extend: {
+      backgroundImage: {
+        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
+        'gradient-conic':
+          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+      },
+    },
+  },
+  plugins: [],
+}

instance-labeler/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}