app/main.py

import os
import io
import base64
import numpy as np
import torch
import time

from PIL import Image
from pydantic import BaseModel
from fastapi import FastAPI
from fastapi.responses import Response, JSONResponse
from fastapi.exceptions import HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
from torchvision.transforms.functional import resize
from .model import (
    build_sam_predictor,
    build_sam_hq_predictor,
    build_mobile_sam_predictor,
    get_multi_label_predictor,
)
from .data import Data
from .configs import DATA_ROOT, DEVICE, MODEL
from .transforms import ResizeLongestSide
from .mobile_sam.utils import batched_mask_to_box


app = FastAPI()
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_methods=["*"],
    allow_headers=["*"],
)
if MODEL == "sam":
    SAM = build_sam_predictor(checkpoint="sam_vit_h_4b8939.pth")
elif MODEL == "sam_hq":
    SAM = build_sam_hq_predictor(checkpoint="sam_hq_vit_h.pth")
elif MODEL == "mobile_sam":
    SAM = build_mobile_sam_predictor(checkpoint="mobile_sam.pth")
else:
    raise ValueError(f"MODEL must be one of sam, sam_hq, got {MODEL}")


DATA = Data(DATA_ROOT / "data.pkl")
T = ResizeLongestSide(1024)


class SamQuery(BaseModel):
    points: list[list[int]]
    labels: list[int]


class MaskLabel(BaseModel):
    mask: str
    label: str


class Masks(BaseModel):
    masks: list[str]


class MaskLabels(BaseModel):
    masks: list[str]
    labels: list[str]


class Box(BaseModel):
    x: int
    y: int
    width: int
    height: int


class Boxes(BaseModel):
    bboxes: list[Box]


class MaskBoxes(BaseModel):
    masks: list[str]
    bboxes: list[Box]


class MaskBoxLabels(BaseModel):
    masks: list[str]
    bboxes: list[Box]
    labels: list[str]


class ImageData(BaseModel):
    image: str


@app.get("/")
async def index():
    return FileResponse(path=f"{os.environ['HOME']}/app/instance-labeler/out/index.html", media_type="text/html")


@app.post("/v1/get_label_preds/{image}")
async def get_label_preds(image: str, q: SamQuery) -> MaskBoxes:
    if image not in DATA:
        raise HTTPException(status_code=404, detail="Image not found")

    if MODEL == "sam" or MODEL == "mobile_sam":
        SAM.features = torch.from_numpy(DATA.get_emb(image)).to(DEVICE)
    elif MODEL == "sam_hq":
        features = DATA.get_hq_emb(image)
        SAM.features = torch.from_numpy(features[0]).to(DEVICE)
        SAM.interm_features = [torch.from_numpy(f).to(DEVICE) for f in features[1:]]
    meta_data = DATA.get_meta_data(image)
    SAM.original_size = meta_data["original_size"]
    SAM.input_size = meta_data["input_size"]
    SAM.is_image_set = True  # type: ignore

    masks, _, _ = SAM.predict(  # type: ignore
        point_coords=np.array(q.points),
        point_labels=np.array(q.labels),
        multimask_output=False,
    )
    bboxes = batched_mask_to_box(torch.as_tensor(masks).to(DEVICE)).cpu().numpy()
    bboxes = [
        Box(x=x1, y=y1, width=y2 - y1, height=x2 - x1)
        for x1, y1, x2, y2 in bboxes.tolist()
    ]
    masks_out = []
    for i in range(masks.shape[0]):
        mask_i = masks[i, :, :]
        mask_i = Image.fromarray(mask_i)
        with io.BytesIO() as buf:
            mask_i.save(buf, format="PNG")
            mask_i = buf.getvalue()
        masks_i_b64 = base64.b64encode(mask_i).decode("utf-8")
        masks_out.append(masks_i_b64)
    return MaskBoxes(masks=masks_out, bboxes=bboxes)


@app.get("/v1/get_labels/{image}")
async def get_labels(image: str) -> MaskBoxLabels:
    if image not in DATA:
        raise HTTPException(status_code=404, detail="Image not found")

    masks, bboxes, labels = DATA.get_labels(image)
    if not masks:
        raise HTTPException(status_code=404, detail="Label not found")

    if len(masks) != len(labels):
        raise HTTPException(
            status_code=400, detail="Currupted data, masks not equal to labels"
        )

    out_masks = []
    for mask in masks:
        with io.BytesIO() as buf:
            mask.save(buf, format="PNG")
            mask = buf.getvalue()
        mask_b64 = base64.b64encode(mask).decode("utf-8")
        out_masks.append(mask_b64)
    bboxes = [Box(x=x1, y=y1, width=w, height=h) for x1, y1, h, w in bboxes]
    return MaskBoxLabels(masks=out_masks, bboxes=bboxes, labels=labels)


@app.post("/v1/get_multi_label_preds/{image}")
async def get_multi_label_preds(image: str, q: MaskLabel) -> MaskBoxLabels:
    if image not in DATA:
        raise HTTPException(status_code=404, detail="Image not found")

    image_pil = DATA.get_image(image)
    image_np = np.array(image_pil.convert("RGB"))
    mask_data = q.mask.replace("data:image/png;base64,", "")
    mask = np.array(Image.open(io.BytesIO(base64.b64decode(mask_data))).convert("L"))

    if mask.sum() == 0:
        raise HTTPException(status_code=422, detail="Mask is empty")

    per_sam_model = get_multi_label_predictor(SAM, image_np, mask)
    start = time.perf_counter()
    masks, bboxes, _ = per_sam_model(image_np)
    print(f"inference time {time.perf_counter() - start}")
    if masks is None:
        return MaskBoxLabels(masks=[], bboxes=[], labels=[])
    masks_out = []
    for i in range(len(masks)):
        mask_i = Image.fromarray(masks[i])
        with io.BytesIO() as buf:
            mask_i.save(buf, format="PNG")
            mask_i = buf.getvalue()
        masks_i_b64 = base64.b64encode(mask_i).decode("utf-8")
        masks_out.append(masks_i_b64)
    bboxes = [
        Box(x=x1, y=y1, width=y2 - y1, height=x2 - x1)
        for x1, y1, x2, y2 in bboxes.tolist()
    ]
    return MaskBoxLabels(
        masks=masks_out, bboxes=bboxes, labels=[q.label for _ in range(len(masks))]
    )


@app.put("/v1/label_image/{image}")
async def label_image(image: str, mask_labels: MaskLabels) -> Response:
    if image not in DATA:
        raise HTTPException(status_code=404, detail="Image not found")

    if len(mask_labels.masks) != len(mask_labels.labels):
        raise HTTPException(status_code=400, detail="Invalid input")

    save_masks = []
    for i in range(len(mask_labels.masks)):
        mask_i = mask_labels.masks[i]
        mask_i = mask_i.replace("data:image/png;base64,", "")
        mask_i = Image.open(io.BytesIO(base64.b64decode(mask_i))).convert("L")
        mask_i = mask_i.point(lambda p: 0 if <= 1 else p)
        save_masks.append(mask_i)
    bboxes = (
        batched_mask_to_box(
            torch.as_tensor(np.array([np.array(m) for m in save_masks]))
            .to(DEVICE)
            .bool()
        )
        .cpu()
        .numpy()
    )
    bboxes = [(x1, y1, (y2 - y1), (x2 - x1)) for x1, y1, x2, y2 in bboxes.tolist()]
    DATA.save_labels(image, save_masks, bboxes, mask_labels.labels)
    return Response(content="saved", media_type="text/plain")


@app.get("/v1/get_image/{image}")
async def get_image(image: str) -> Response:
    if image not in DATA:
        raise HTTPException(status_code=404, detail="Image not found")

    image_ = DATA.get_image(image)

    if not DATA.emb_exists(image):
        SAM.set_image(np.asarray(image_.convert("RGB")))  # type: ignore
        if MODEL == "sam" or MODEL == "mobile_sam":
            features = SAM.get_image_embedding().detach().cpu().numpy()  # type: ignore
            DATA.save_emb(image, features)
        elif MODEL == "sam_hq":
            features = [SAM.features] + SAM.interm_features  # type: ignore
            DATA.save_hq_emb(image, [f.detach().cpu().numpy() for f in features])
        DATA.save_meta_data(
            image,
            {"original_size": SAM.original_size, "input_size": SAM.input_size},
        )

    with io.BytesIO() as buf:
        image_.save(buf, format="PNG")
        image_ = buf.getvalue()
    image_b64 = base64.b64encode(image_).decode("utf-8")
    return Response(content=image_b64, media_type="image/png")


@app.put("/v1/upload_image/{image}")
async def upload_image(image: str, image_data: ImageData) -> Response:
    image_b64 = image_data.image
    image_b64 = image_b64.replace("data:image/png;base64,", "")
    image_b64 = image_b64.replace("data:image/jpeg;base64,", "")
    if "data:image/" in image_b64:
        raise HTTPException(
            status_code=400, detail="Invalid image format, only accepts png and jpeg"
        )

    image_pil = Image.open(io.BytesIO(base64.b64decode(image_b64))).convert("RGB")
    # image_bytes = io.BytesIO(base64.b64decode(image_b64))
    # image_pil = Image.open(image_bytes)
    target_size = T.get_preprocess_shape(
        image_pil.size[1], image_pil.size[0], T.target_length
    )
    image_pil = resize(image_pil, target_size)
    image_id = DATA.save_image(image, image_pil)
    return Response(content=image_id, media_type="text/plain")


@app.get("/v1/get_all_images")
async def get_all_images() -> Response:
    return JSONResponse(content={"images": DATA.get_all_images()})


app.mount("/", StaticFiles(directory=f"{os.environ['HOME']}/app/instance-labeler/out", html=True), name="static")