init

.gitattributes

@@ -32,3 +32,88 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/animals.png filter=lfs diff=lfs merge=lfs -text
+images/region_retrieval.png filter=lfs diff=lfs merge=lfs -text
+xdecoder_focalt_last_novg.pt filter=lfs diff=lfs merge=lfs -text
+xdecoder_focalt_last.pt filter=lfs diff=lfs merge=lfs -text
+v_emb.da filter=lfs diff=lfs merge=lfs -text
+images/coco/077.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/071.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/022.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/026.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/036.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/039.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/052.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/057.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/061.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/017.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/021.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/030.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/056.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/064.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/072.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/014.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/025.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/027.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/038.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/044.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/049.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/053.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/078.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/002.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/005.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/007.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/008.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/011.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/013.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/020.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/034.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/000.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/066.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/074.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/065.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/023.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/024.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/033.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/040.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/041.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/046.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/060.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/003.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/058.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/073.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/042.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/015.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/016.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/018.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/051.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/054.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/063.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/010.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/050.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/070.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/037.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/031.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/062.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/067.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/069.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/029.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/012.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/068.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/075.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/006.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/035.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/001.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/055.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/048.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/019.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/028.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/043.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/059.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/079.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/004.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/032.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/045.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/047.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/076.jpg filter=lfs diff=lfs merge=lfs -text
+images/coco/009.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,103 @@
+# IntelliJ project files
+.idea
+*.iml
+out
+gen
+
+### Vim template
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~
+
+### IPythonNotebook template
+# Temporary data
+.ipynb_checkpoints/
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+#lib/
+#lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+*.ipynb
+*.params
+# *.json
+.vscode/
+*.code-workspace/
+
+lib/pycocotools/_mask.c
+lib/nms/cpu_nms.c
+
+OUTPUT
+OUTPUT/*
+models/*
+DATASET
+DATASET/*
+external/
+MODELS
+MODELS/*
+gradio_cached_examples/*
+
+kill.sh
+
+draws/
+plot/
+
+*venv/*

README.md

@@ -1,12 +1,13 @@
 ---
-title: Demo
-emoji: 🐢
+title: X Decoder
+emoji: 📈
 colorFrom: purple
-colorTo: red
+colorTo: gray
 sdk: gradio
-sdk_version: 3.15.0
+sdk_version: 3.14.0
 app_file: app.py
 pinned: false
+license: afl-3.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,120 @@
+# --------------------------------------------------------
+# X-Decoder -- Generalized Decoding for Pixel, Image, and Language
+# Copyright (c) 2022 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Xueyan Zou (xueyan@cs.wisc.edu), Jianwei Yang (jianwyan@microsoft.com)
+# --------------------------------------------------------
+
+import os
+os.system("python -m pip install git+https://github.com/MaureenZOU/detectron2-xyz.git")
+
+import gradio as gr
+import torch
+import argparse
+
+from xdecoder.BaseModel import BaseModel
+from xdecoder import build_model
+from utils.distributed import init_distributed
+from utils.arguments import load_opt_from_config_files
+
+from tasks import *
+
+def parse_option():
+    parser = argparse.ArgumentParser('X-Decoder All-in-One Demo', add_help=False)
+    parser.add_argument('--conf_files', default="configs/xdecoder/svlp_focalt_lang.yaml", metavar="FILE", help='path to config file', )
+    args = parser.parse_args()
+
+    return args
+
+'''
+build args
+'''
+args = parse_option()
+opt = load_opt_from_config_files(args.conf_files)
+opt = init_distributed(opt)
+
+# META DATA
+pretrained_pth_last = os.path.join("xdecoder_focalt_last.pt")
+pretrained_pth_novg = os.path.join("xdecoder_focalt_last_novg.pt")
+
+if not os.path.exists(pretrained_pth_last):
+    os.system("wget {}".format("https://projects4jw.blob.core.windows.net/x-decoder/release/xdecoder_focalt_last.pt"))
+
+if not os.path.exists(pretrained_pth_novg):
+    os.system("wget {}".format("https://projects4jw.blob.core.windows.net/x-decoder/release/xdecoder_focalt_last_novg.pt"))
+
+
+'''
+build model
+'''
+model_last = BaseModel(opt, build_model(opt)).from_pretrained(pretrained_pth_last).eval().cuda()
+model_cap = BaseModel(opt, build_model(opt)).from_pretrained(pretrained_pth_novg).eval().cuda()
+
+with torch.no_grad():
+    model_last.model.sem_seg_head.predictor.lang_encoder.get_text_embeddings(["background", "background"], is_eval=True)
+    model_cap.model.sem_seg_head.predictor.lang_encoder.get_text_embeddings(["background", "background"], is_eval=True)
+
+'''
+inference model
+'''
+
+@torch.no_grad()
+def inference(image, task, *args, **kwargs):
+    image = image.convert("RGB")
+    with torch.autocast(device_type='cuda', dtype=torch.float16):
+        if task == 'Referring Inpainting':
+            return referring_inpainting(model_last, image, *args, **kwargs)
+        elif task == 'Referring Segmentation':
+            return referring_segmentation(model_last, image, *args, **kwargs)
+        elif task == 'Open Vocabulary Semantic Segmentation':
+            return open_semseg(model_last, image, *args, **kwargs)
+        elif task == 'Open Vocabulary Panoptic Segmentation':
+            return open_panoseg(model_last, image, *args, **kwargs)
+        elif task == 'Open Vocabulary Instance Segmentation':
+            return open_instseg(model_last, image, *args, **kwargs)
+        elif task == 'Image Captioning':
+            return image_captioning(model_cap, image, *args, **kwargs)
+        elif task == 'Referring Captioning (Beta)':
+            return referring_captioning([model_last, model_cap], image, *args, **kwargs)
+        elif task == 'Text Retrieval':
+            return text_retrieval(model_cap, image, *args, **kwargs)
+        elif task == 'Image/Region Retrieval (Only Support Exampled 80 images)':
+            return region_retrieval([model_cap, model_last], image, *args, **kwargs)
+
+'''
+launch app
+'''
+title = "X-Decoder All-in-One Demo"
+description = "<p style='text-align: center'> <a href='' target='_blank'>Project Page</a> | <a href='' target='_blank'>Paper</a> | <a href='https://github.com/microsoft/X-Decoder' target='_blank'>Github Repo</a> | <a href='' target='_blank'>Video</a> </p>"
+article = "The Demo is Run on X-Decoder (Focal-T)."
+
+inputs = [gr.inputs.Image(type='pil'), gr.inputs.Radio(choices=["Referring Segmentation", 'Open Vocabulary Semantic Segmentation','Open Vocabulary Instance Segmentation', "Open Vocabulary Panoptic Segmentation", "Image Captioning", "Text Retrieval", "Referring Inpainting", "Referring Captioning (Beta)", "Image/Region Retrieval (Only Support Exampled 80 images)"], type="value", default="OpenVocab Semantic Segmentation", label="Task"), gr.Textbox(label="xdecoder_text"), gr.Textbox(label="inpainting_text"), gr.Textbox(label="task_description")]
+gr.Interface(
+    fn=inference,
+    inputs=inputs,
+    outputs=[
+        gr.outputs.Image(
+        type="pil",
+        label="segmentation results"),
+        gr.Textbox(label="text restuls"),
+        gr.outputs.Image(
+        type="pil",
+        label="inpainting results"),
+    ],
+    examples=[
+    ["./images/fruit.jpg", "Referring Segmentation", "The larger watermelon.,The front white flower.,White tea pot.,Flower bunch.,white vase.,The peach on the left.,The brown knife.", '', 'Format: s,s,s'],
+    ["./images/animals.png", "Open Vocabulary Semantic Segmentation", "zebra,antelope,giraffe,ostrich,sky,water,grass,sand,tree", '', 'Format: x,x,x'],
+    ["./images/street.jpg", "Open Vocabulary Panoptic Segmentation", "stuff:building,sky,street,tree,rock,sidewalk;thing:car,person,traffic light", '', 'Format: stuff:x,x,x;thing:y,y,y'],
+    ["./images/owls.jpeg", "Open Vocabulary Instance Segmentation", "owl", '', 'Format: y,y,y'],
+    ["./images/mountain.jpeg", "Image Captioning", "", '', ''],
+    ["./images/rose.webp", "Text Retrieval", "lily,rose,peoney,tulip", '', 'Format: s,s,s'],
+    ["./images/region_retrieval.png", "Image/Region Retrieval (Only Support Exampled 80 images)", "The tangerine on the plate.", '', 'Please describe the object in a detailed way.'],
+    ["./images/landscape.jpg", "Referring Captioning (Beta)", "cloud", '', 'Please fill in a noun/noun phrase. (may start with a/the)'],
+    ["./images/apples.jpg", "Referring Inpainting", "a yellow apple", 'a pear', 'x-decoder + ldm (inference takes ~40s.)'],
+    ],
+    title=title,
+    description=description,
+    article=article,
+    allow_flagging='never',
+    cache_examples=True,
+).launch(share=True)

configs/xdecoder/svlp_focalt_lang.yaml

@@ -0,0 +1,110 @@
+# --------------------------------------------------------
+# X-Decoder -- Generalized Decoding for Pixel, Image, and Language
+# Copyright (c) 2022 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Xueyan Zou (xueyan@cs.wisc.edu)
+# --------------------------------------------------------
+
+##################
+# Task settings
+##################
+VERBOSE: true
+MODEL:
+  NAME: xdecoder_model
+  HEAD: xdecoder_head
+  DIM_PROJ: 512
+  BACKBONE_DIM: 768
+  TEXT:
+    ARCH: vlpencoder
+    NAME: transformer
+    TOKENIZER: clip
+    CONTEXT_LENGTH: 77 # 77
+    WIDTH: 512
+    HEADS: 8
+    LAYERS: 12 # 6
+    AUTOGRESSIVE: True
+  BACKBONE:
+    NAME: focal_dw
+    PRETRAINED: ''
+    LOAD_PRETRAINED: false
+    FOCAL:
+      PRETRAIN_IMG_SIZE: 224
+      PATCH_SIZE: 4
+      EMBED_DIM: 96
+      DEPTHS: [2, 2, 6, 2]
+      FOCAL_LEVELS: [3, 3, 3, 3]
+      FOCAL_WINDOWS: [3, 3, 3, 3]
+      DROP_PATH_RATE: 0.3
+      MLP_RATIO: 4.0
+      DROP_RATE: 0.0
+      PATCH_NORM: True
+      USE_CONV_EMBED: True
+      SCALING_MODULATOR: True
+      USE_CHECKPOINT: False
+      USE_POSTLN: true
+      USE_POSTLN_IN_MODULATION: false
+      USE_LAYERSCALE: True
+      OUT_FEATURES: ["res2", "res3", "res4", "res5"]
+      OUT_INDICES: [0, 1, 2, 3]
+  ENCODER:
+    NAME: transformer_encoder_fpn
+    IGNORE_VALUE: 255
+    NUM_CLASSES: 133
+    LOSS_WEIGHT: 1.0
+    CONVS_DIM: 512
+    MASK_DIM: 512
+    NORM: "GN"
+    IN_FEATURES: ["res2", "res3", "res4", "res5"]
+    DEFORMABLE_TRANSFORMER_ENCODER_IN_FEATURES: ["res3", "res4", "res5"]
+    COMMON_STRIDE: 4
+    TRANSFORMER_ENC_LAYERS: 6
+  DECODER:
+    NAME: xdecoder
+    TRANSFORMER_IN_FEATURE: "multi_scale_pixel_decoder"
+    MASK: True
+    GROUNDING:
+      ENABLED: True
+      MAX_LEN: 5
+      TEXT_WEIGHT: 2.0
+      CLASS_WEIGHT: 0.5
+    DETECTION: False
+    CAPTION:
+      ENABLED: True
+      PHRASE_PROB: 0.0
+      SIM_THRES: 0.95
+    CAPTIONING:
+      ENABLED: True
+      STEP: 50
+    RETRIEVAL:
+      ENABLED: True
+      DIM_IMG: 768
+      ENSEMBLE: True
+    HIDDEN_DIM: 512
+    NUM_OBJECT_QUERIES: 101
+    NHEADS: 8
+    DROPOUT: 0.0
+    DIM_FEEDFORWARD: 2048
+    PRE_NORM: False
+    ENFORCE_INPUT_PROJ: False
+    SIZE_DIVISIBILITY: 32
+    TRAIN_NUM_POINTS: 12544
+    OVERSAMPLE_RATIO: 3.0
+    IMPORTANCE_SAMPLE_RATIO: 0.75
+    DEC_LAYERS: 10  # 9 decoder layers, add one for the loss on learnable query
+    TOP_GROUNDING_LAYERS: 3
+    TOP_CAPTION_LAYERS: 3
+    TOP_CAPTIONING_LAYERS: 3
+    TOP_RETRIEVAL_LAYERS: 3
+    TOP_OPENIMAGE_LAYERS: 10
+    TEST:
+      SEMANTIC_ON: True
+      INSTANCE_ON: True
+      PANOPTIC_ON: True
+      OVERLAP_THRESHOLD: 0.8
+      OBJECT_MASK_THRESHOLD: 0.4
+      SEM_SEG_POSTPROCESSING_BEFORE_INFERENCE: false
+      DETECTIONS_PER_IMAGE: 100
+
+INPUT:
+  PIXEL_MEAN: [123.675, 116.280, 103.530]
+  PIXEL_STD: [58.395, 57.120, 57.375]