File size: 5,051 Bytes
907ff09 cc277bb 907ff09 50f8c72 6deaf33 907ff09 c94e841 cc277bb c94e841 907ff09 0a7a87a 907ff09 6deaf33 e0c63a6 cc277bb 907ff09 63f30af 907ff09 0a7a87a 63f30af cc277bb 907ff09 cc277bb 907ff09 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
import os
from pyChatGPT import ChatGPT
#import cloudpickle
os.system("pip install -U gradio")
import sys
import gradio as gr
os.system(
"pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html"
)
# clone and install Detic
os.system(
"git clone https://github.com/facebookresearch/Detic.git --recurse-submodules"
)
os.chdir("Detic")
# Install detectron2
import torch
# Some basic setup:
# Setup detectron2 logger
import detectron2
from detectron2.utils.logger import setup_logger
setup_logger()
# import some common libraries
import sys
import numpy as np
import os, json, cv2, random
# import some common detectron2 utilities
from detectron2 import model_zoo
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog, DatasetCatalog
# Detic libraries
sys.path.insert(0, "third_party/CenterNet2/projects/CenterNet2/")
sys.path.insert(0, "third_party/CenterNet2/")
from centernet.config import add_centernet_config
from detic.config import add_detic_config
from detic.modeling.utils import reset_cls_test
from PIL import Image
# Build the detector and download our pretrained weights
cfg = get_cfg()
add_centernet_config(cfg)
add_detic_config(cfg)
cfg.MODEL.DEVICE = "cpu"
cfg.merge_from_file("configs/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.yaml")
cfg.MODEL.WEIGHTS = "https://dl.fbaipublicfiles.com/detic/Detic_LCOCOI21k_CLIP_SwinB_896b32_4x_ft4x_max-size.pth"
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 # set threshold for this model
cfg.MODEL.ROI_BOX_HEAD.ZEROSHOT_WEIGHT_PATH = "rand"
cfg.MODEL.ROI_HEADS.ONE_CLASS_PER_PROPOSAL = (
True # For better visualization purpose. Set to False for all classes.
)
predictor = DefaultPredictor(cfg)
# Setup the model's vocabulary using build-in datasets
BUILDIN_CLASSIFIER = {
"lvis": "datasets/metadata/lvis_v1_clip_a+cname.npy",
"objects365": "datasets/metadata/o365_clip_a+cnamefix.npy",
"openimages": "datasets/metadata/oid_clip_a+cname.npy",
"coco": "datasets/metadata/coco_clip_a+cname.npy",
}
BUILDIN_METADATA_PATH = {
"lvis": "lvis_v1_val",
"objects365": "objects365_v2_val",
"openimages": "oid_val_expanded",
"coco": "coco_2017_val",
}
vocabulary = "lvis" # change to 'lvis', 'objects365', 'openimages', or 'coco'
metadata = MetadataCatalog.get(BUILDIN_METADATA_PATH[vocabulary])
classifier = BUILDIN_CLASSIFIER[vocabulary]
num_classes = len(metadata.thing_classes)
reset_cls_test(predictor.model, classifier, num_classes)
def inference(img,unique_only):
im = cv2.imread(img)
outputs = predictor(im)
#v = Visualizer(im[:, :, ::-1], metadata)
#out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
detected_objects = []
object_list_str = []
#masks = outputs["instances"].pred_masks
#masks_screen = masks.cpu().numpy()
#print(masks_screen)
box_locations = outputs["instances"].pred_boxes
box_loc_screen = box_locations.tensor.cpu().numpy()
unique_object_dict = {}
for i, box_coord in enumerate(box_loc_screen):
x0, y0, x1, y1 = box_coord
width = x1 - x0
height = y1 - y0
predicted_label = metadata.thing_classes[outputs["instances"].pred_classes[i]]
detected_objects.append(
{
"prediction": predicted_label,
"x": int(x0),
"y": int(y0),
"width": int(width),
"height": int(height),
}
)
if ((not unique_only) or (unique_only and predicted_label not in unique_object_dict)):
object_list_str.append(
f"{predicted_label} - X:{int(x0)} Y: {int(y0)} Width: {int(width)} Height: {int(height)}"
)
unique_object_dict[predicted_label] = 1
output_str = "Imagine you are a blind but intelligent image captioner who is only given the X,Y coordinates and width, height of each object in a scene with no specific attributes of the objects themselves. Create a description of the scene using the relative positions and sizes of objects\n"
for line in object_list_str:
output_str += line + "\n"
return (
#Image.fromarray(np.uint8(out.get_image())).convert("RGB"),
detected_objects
#cloudpickle.dumps(masks_screen)
)
with gr.Blocks() as demo:
with gr.Column():
inp = gr.Image(label="Input Image", type="filepath")
chk = gr.Checkbox(label="Unique Objects only? (useful to reduce ChatGPT input to speed up its response and also eliminate timeouts")
btn_detic = gr.Button("Run Detic for ChatGPT")
with gr.Column():
#outviz = gr.Image(label="Visualization", type="pil")
output_desc = gr.JSON(label="Detected Objects")
#masks_screen = gr.Textbox(label="Masks binary", lines=1)
btn_detic.click(fn=inference, inputs=[inp,chk], outputs=[output_desc], api_name="detect")
demo.launch()
|