feat: zeroGPU spaces support (drop docker, uses gradio sdk)

Dockerfile

@@ -1,61 +0,0 @@
-FROM nvcr.io/nvidia/pytorch:24.03-py3
-
-LABEL authors="alessandro@trinca.tornidor.com"
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG WORKDIR="/var/task"
-
-ENV PYTHONUNBUFFERED=1
-ENV PYTHONPATH=${WORKDIR}:${WORKDIR}/venv:${PYTHONPATH}
-ENV PATH=${WORKDIR}/venv/bin:$PATH
-ENV XDG_CACHE_HOME=/data
-
-WORKDIR ${WORKDIR}
-COPY . ${WORKDIR}/
-RUN ls ${WORKDIR}/
-RUN mkdir -p ${XDG_CACHE_HOME}/.cache
-RUN chmod 770 ${XDG_CACHE_HOME}/.cache
-
-RUN apt update && apt upgrade -y && apt install --no-install-recommends -y \
-  build-essential \
-  python3.11 \
-  python3-pip \
-  python3-dev \
-  python3-venv \
-  git \
-  ffmpeg \
-  curl \
-  && apt clean && rm -rf /var/lib/apt/lists/*
-
-RUN which python3
-RUN python3 --version
-RUN python3 -m venv venv
-RUN source ${WORKDIR}/venv/bin/activate python -m pip install pip --upgrade && python -m pip install -r ${WORKDIR}/requirements.txt
-RUN source ${WORKDIR}/venv/bin/activate && which python && python --version
-RUN chmod +x ${WORKDIR}/scripts/entrypoint.sh
-RUN curl -o /tmp/frpc_linux_amd64_v0.2 https://cdn-media.huggingface.co/frpc-gradio-0.2/frpc_linux_amd64
-RUN ls -l /tmp/frpc_linux_amd64_v0.2
-RUN cp /tmp/frpc_linux_amd64_v0.2 ${WORKDIR}/venv/lib/python*/site-packages/gradio
-RUN ls -l ${WORKDIR}/venv/lib/python*/site-packages/gradio
-RUN ls -l ${WORKDIR}/venv/bin
-RUN bash --version
-RUN chmod 770 ${WORKDIR}/flagged/
-RUN chmod 770 ${WORKDIR}/flagged/* || true
-RUN ls -ld ${WORKDIR}/flagged/
-RUN ls -ld ${WORKDIR}/flagged/* || echo "folders ${WORKDIR}/flagged/* not found"
-RUN ls -l ${WORKDIR}
-RUN ls -l ${WORKDIR}/scripts/
-RUN ls -l ${WORKDIR}/scripts/entrypoint.sh
-
-EXPOSE 7860
-
-CMD ["/var/task/scripts/entrypoint.sh"]
-# CMD [
-#  "/var/task/scripts/entrypoint.sh",
-#  "/var/task/venv/bin/uvicorn", "app:lisa_app",
-#  "--host", "0.0.0.0",
-#  "--port", "7860",
-#  "--version='xinlai/LISA-13B-llama2-v1-explanatory'",
-#  "--precision='fp16'",
-#  "--load_in_4bit"
-# ]

README.md

@@ -1,20 +1,25 @@
 ---
-title: LISA On Cuda
-emoji: 📊
-colorFrom: yellow
-colorTo: red
-sdk: docker
-pinned: false
+title: lisa + gradio + fastapi + ZeroGPU
+emoji: ⚡
+colorFrom: red
+colorTo: purple
+sdk: gradio
+sdk_version: 4.37.2
+app_file: app.py
+pinned: true
 ---
 
-# exec jupyter on the remote server with port forwarding on localhost
+# LISA (Reasoning Segmentation via Large Language Model) on cuda, now with huggingface ZeroGPU support!
+
+## Exec jupyter on the remote server with port forwarding on localhost
 
 1. checkout repo, install venv with jupyter
 2. port forwarding in localhost wiht private key: `ssh -i /path/to/private_key name@endpoint.com -L 8889:localhost:8889 -N -f`
 3. start the jupyter-lab server
 4. connect to page in localhost
 
-## Commands to work on saturncloud after clone and git lfs install
+## Commands to work on remote virtual machines (e.g. SaturnCloud) after clone and git lfs install
+
 ```bash
 cd ~/workspace/lisa-on-cuda/
 rm -rf lisa_venv 
@@ -38,320 +43,17 @@ To run the `test.ipynb` notebook you should already:
 - installed jupyterlab dependencies from requirements_jupyter.txt
 - installed dependencies from requirements.txt
 
-## Hardware requirements
+## Hardware requirements for local usage
+
 - an nvidia gpu with 10 or 12GB of memory (a T4 should suffice)
 - at least 16GB of system ram
 
-[![Gradio](https://img.shields.io/badge/Gradio-Online%20Demo-blue)](http://103.170.5.190:7860/)
-[![Open in OpenXLab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/openxlab-app/LISA)
-
-# LISA: Reasoning Segmentation via Large Language Model
-
-<font size=7><div align='center'><b>LISA</b>: Large <b>L</b>anguage <b>I</b>nstructed <b>S</b>egmentation <b>A</b>ssistant</div></font>
-
-<font size=7><div align='center' > <a href=https://arxiv.org/pdf/2308.00692.pdf>**Paper**</a> | <a href="https://huggingface.co/xinlai">**Models**</a> | [**Training**](#training) | [**Inference**](#inference) | [**Local Deployment**](#deployment) | [**Dataset**](#dataset) | <a href="http://103.170.5.190:7860/">**Online Demo**</a></div></font>
-
-<!-- <p align="center"> <img src="resources/imgs/teaser.jpg" width="100%"> </p> -->
-
-<table class="center">
-<tr>
-  <td style="text-align:center;"><b>Input</b></td>
-  <td style="text-align:center;"><b>Output</b></td>
-  <td style="text-align:center;"><b>Input</b></td>
-  <td style="text-align:center;"><b>Output</b></td>
-</tr>
-<tr>
-  <td><img src="resources/imgs/obama.jpg"></td>
-  <td><img src="vis_output/obama.jpg"></td>
-  <td><img src="resources/imgs/trump.jpg"></td>              
-  <td><img src="vis_output/trump.jpg"></td>
-</tr>
-<tr>
-  <td width=25% style="text-align:center;color:gray;">"Who was <ins>the president of the US</ins> in this image? Please output segmentation mask and explain the reason."</td>
-  <td width=25% style="text-align:center;">"Sure, the segmentation result is [SEG]. The President of the United States in the image is <ins>President Obama.</ins>”</td>
-  <td width=25% style="text-align:center;color:gray">"Who was <ins>the president of the US</ins> in this image? Please output segmentation mask and explain why."</td>
-  <td width=25% style="text-align:center;">"Sure, [SEG]. In the image, the President of the United States is <ins>President Trump.</ins>"</td>
-</tr>
-
-<tr>
-  <td><img src="resources/imgs/stand_higher.jpg"></td>
-  <td><img src="vis_output/stand_higher.jpg"></td>
-  <td><img src="resources/imgs/camera_lens.jpg"></td>              
-  <td><img src="vis_output/camera_lens.jpg"></td>
-</tr>
-<tr>
-  <td width=25% style="text-align:center;color:gray;">"What can <ins>make the woman stand higher</ins>? Please output segmentation mask and explain why."</td>
-  <td width=25% style="text-align:center;">"Sure, [SEG]. The woman is <ins>standing higher by using a ladder</ins>..."</td>
-  <td width=25% style="text-align:center;color:gray">"Can you segment <ins>the camera lens that is more suitable for photographing nearby objects</ins> in this image?"</td>
-  <td width=25% style="text-align:center;">"Sure, [SEG]."</td>
-</tr>
-
-<tr>
-  <td><img src="resources/imgs/dog_with_horn.jpg"></td>
-  <td><img src="vis_output/dog_with_horn.jpg"></td>
-  <td><img src="resources/imgs/wash_hands.jpg"></td>              
-  <td><img src="vis_output/wash_hands.jpg"></td>
-</tr>
-<tr>
-  <td width=25% style="text-align:center;color:gray;">"Can you segment <ins>the unusual part</ins> in this image and explain why."</td>
-  <td width=25% style="text-align:center;">"Sure, it is [SEG]. In the image, the unusual part is <ins>the dog wearing a reindeer antler headband</ins>..."</td>
-  <td width=25% style="text-align:center;color:gray">"Where to <ins>wash hands</ins> in this image? Please output segmentation mask."</td>
-  <td width=25% style="text-align:center;">"Sure, [SEG]."</td>
-</tr>
-
-<tr>
-  <td><img src="resources/imgs/jackma.jpg"></td>
-  <td><img src="vis_output/jackma.jpg"></td>
-  <td><img src="resources/imgs/blackpink.jpg"></td>              
-  <td><img src="vis_output/blackpink.jpg"></td>
-</tr>
-<tr>
-  <td width=25% style="text-align:center;color:gray;">"Can you segment <ins>the founder of Alibaba</ins> in this image and explain why?"</td>
-  <td width=25% style="text-align:center;">"Sure, it is [SEG]. In the image, the man wearing a red tie and sitting on one of the chairs is <ins>Jack Ma</ins>, the co-founder of Alibaba Group..."</td>
-  <td width=25% style="text-align:center;color:gray">"Please segment <ins>Lisa</ins> in this figure."</td>
-  <td width=25% style="text-align:center;">"Sure, [SEG]."</td>
-</tr>
-</table>
-
-<p align="center"> <img src="resources/imgs/fig_overview.jpg" width="100%"> </p>
-
-## News
-- [x] [2023.8.30] Release three new models [LISA-7B-v1](https://huggingface.co/xinlai/LISA-7B-v1), [LISA-7B-v1-explanatory](https://huggingface.co/xinlai/LISA-7B-v1-explanatory), and [LISA-13B-llama2-v1-explanatory](https://huggingface.co/xinlai/LISA-13B-llama2-v1-explanatory). Welcome to check them out!
-- [x] [2023.8.23] Refactor code, and release new model [LISA-13B-llama2-v1](https://huggingface.co/xinlai/LISA-13B-llama2-v1). Welcome to check it out!
-- [x] [2023.8.9] Training code is released!
-- [x] [2023.8.4] [Online Demo](http://103.170.5.190:7860/) is released! 
-- [x] [2023.8.4] [*ReasonSeg* Dataset](https://drive.google.com/drive/folders/125mewyg5Ao6tZ3ZdJ-1-E3n04LGVELqy?usp=sharing) and the [LISA-13B-llama2-v0-explanatory](https://huggingface.co/xinlai/LISA-13B-llama2-v0-explanatory) model are released! 
-- [x] [2023.8.3] Inference code and the [LISA-13B-llama2-v0](https://huggingface.co/xinlai/LISA-13B-llama2-v0) model are released. Welcome to check them out!
-- [x] [2023.8.2] [Paper](https://arxiv.org/pdf/2308.00692.pdf) is released and GitHub repo is created.
-
-**LISA: Reasoning Segmentation via Large Language Model [[Paper](https://arxiv.org/abs/2308.00692)]** <br />
-[Xin Lai](https://scholar.google.com/citations?user=tqNDPA4AAAAJ&hl=zh-CN),
-[Zhuotao Tian](https://scholar.google.com/citations?user=mEjhz-IAAAAJ&hl=en),
-[Yukang Chen](https://scholar.google.com/citations?user=6p0ygKUAAAAJ&hl=en),
-[Yanwei Li](https://scholar.google.com/citations?user=I-UCPPcAAAAJ&hl=zh-CN),
-[Yuhui Yuan](https://scholar.google.com/citations?user=PzyvzksAAAAJ&hl=en),
-[Shu Liu](https://scholar.google.com.hk/citations?user=BUEDUFkAAAAJ&hl=zh-CN),
-[Jiaya Jia](https://scholar.google.com/citations?user=XPAkzTEAAAAJ&hl=en)<br />
-
-## Abstract
-In this work, we propose a new segmentation task --- ***reasoning segmentation***. The task is designed to output a segmentation mask given a complex and implicit query text. We establish a benchmark comprising over one thousand image-instruction pairs, incorporating intricate reasoning and world knowledge for evaluation purposes. Finally, we present LISA: Large-language Instructed Segmentation Assistant, which inherits the language generation capabilities of the multi-modal Large Language Model (LLM) while also possessing the ability to produce segmentation masks.
-For more details, please refer to the [paper](https://arxiv.org/abs/2308.00692).
-
-## Highlights
-**LISA** unlocks the new segmentation capabilities of multi-modal LLMs, and can handle cases involving: 
-1. complex reasoning; 
-2. world knowledge; 
-3. explanatory answers; 
-4. multi-turn conversation. 
-
-**LISA** also demonstrates robust zero-shot capability when trained exclusively on reasoning-free datasets. In addition, fine-tuning the model with merely 239 reasoning segmentation image-instruction pairs results in further performance enhancement.
-
-## Experimental results
-<p align="center"> <img src="resources/imgs/table1.jpg" width="80%"> </p>
-
-## Installation
-```
-pip install -r requirements.txt
-pip install flash-attn --no-build-isolation
-```
-
-## Training
-### Training Data Preparation
-The training data consists of 4 types of data:
-
-1. Semantic segmentation datasets: [ADE20K](http://data.csail.mit.edu/places/ADEchallenge/ADEChallengeData2016.zip), [COCO-Stuff](http://calvin.inf.ed.ac.uk/wp-content/uploads/data/cocostuffdataset/stuffthingmaps_trainval2017.zip), [Mapillary](https://www.mapillary.com/dataset/vistas), [PACO-LVIS](https://github.com/facebookresearch/paco/tree/main#dataset-setup), [PASCAL-Part](https://github.com/facebookresearch/VLPart/tree/main/datasets#pascal-part), [COCO Images](http://images.cocodataset.org/zips/train2017.zip)
-
-    Note: For COCO-Stuff, we use the annotation file stuffthingmaps_trainval2017.zip. We only use the PACO-LVIS part in PACO. COCO Images should be put into the `dataset/coco/` directory.
-
-3. Referring segmentation datasets: [refCOCO](https://web.archive.org/web/20220413011718/https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcoco.zip), [refCOCO+](https://web.archive.org/web/20220413011656/https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcoco+.zip), [refCOCOg](https://web.archive.org/web/20220413012904/https://bvisionweb1.cs.unc.edu/licheng/referit/data/refcocog.zip), [refCLEF](https://web.archive.org/web/20220413011817/https://bvisionweb1.cs.unc.edu/licheng/referit/data/refclef.zip) ([saiapr_tc-12](https://web.archive.org/web/20220515000000/http://bvisionweb1.cs.unc.edu/licheng/referit/data/images/saiapr_tc-12.zip)) 
-
-    Note: the original links of refCOCO series data are down, and we update them with new ones. If the download speed is super slow or unstable, we also provide a [OneDrive link](https://mycuhk-my.sharepoint.com/:f:/g/personal/1155154502_link_cuhk_edu_hk/Em5yELVBvfREodKC94nOFLoBLro_LPxsOxNV44PHRWgLcA?e=zQPjsc) to download. **You must also follow the rules that the original datasets require.**
-
-4. Visual Question Answering dataset: [LLaVA-Instruct-150k](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_150k.json)
-
-5. Reasoning segmentation dataset: [ReasonSeg](https://github.com/dvlab-research/LISA#dataset)
-
-Download them from the above links, and organize them as follows.
-
-```
-├── dataset
-│         ├── ade20k
-│         │         ├── annotations
-│         │         └── images
-│         ├── coco
-│         │         └── train2017
-│         │             ├── 000000000009.jpg
-│         │             └── ...
-│         ├── cocostuff
-│         │         └── train2017
-│         │             ├── 000000000009.png
-│         │             └── ...
-│         ├── llava_dataset
-│         │         └── llava_instruct_150k.json
-│         ├── mapillary
-│         │         ├── config_v2.0.json
-│         │         ├── testing
-│         │         ├── training
-│         │         └── validation
-│         ├── reason_seg
-│         │         └── ReasonSeg
-│         │             ├── train
-│         │             ├── val
-│         │             └── explanatory
-│         ├── refer_seg
-│         │         ├── images
-│         │         |   ├── saiapr_tc-12 
-│         │         |   └── mscoco
-│         │         |       └── images
-│         │         |           └── train2014
-│         │         ├── refclef
-│         │         ├── refcoco
-│         │         ├── refcoco+
-│         │         └── refcocog
-│         └── vlpart
-│             ├── paco
-│       │   └── annotations
-│             └── pascal_part
-│                 ├── train.json
-│           └── VOCdevkit
-```
-
-### Pre-trained weights
-
-#### LLaVA
-To train LISA-7B or 13B, you need to follow the [instruction](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md) to merge the LLaVA delta weights. Typically, we use the final weights `LLaVA-Lightning-7B-v1-1` and `LLaVA-13B-v1-1` merged from `liuhaotian/LLaVA-Lightning-7B-delta-v1-1` and `liuhaotian/LLaVA-13b-delta-v1-1`, respectively. For Llama2, we can directly use the LLaVA full weights `liuhaotian/llava-llama-2-13b-chat-lightning-preview`.
-
-#### SAM ViT-H weights
-Download SAM ViT-H pre-trained weights from the [link](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth).
-
-### Training
-```
-deepspeed --master_port=24999 train_ds.py \
-  --version="PATH_TO_LLaVA" \
-  --dataset_dir='./dataset' \
-  --vision_pretrained="PATH_TO_SAM" \
-  --dataset="sem_seg||refer_seg||vqa||reason_seg" \
-  --sample_rates="9,3,3,1" \
-  --exp_name="lisa-7b"
-```
-When training is finished, to get the full model weight:
-```
-cd ./runs/lisa-7b/ckpt_model && python zero_to_fp32.py . ../pytorch_model.bin
-```
-
-### Merge LoRA Weight
-Merge the LoRA weights of `pytorch_model.bin`, save the resulting model into your desired path in the Hugging Face format:
-```
-CUDA_VISIBLE_DEVICES="" python merge_lora_weights_and_save_hf_model.py \
-  --version="PATH_TO_LLaVA" \
-  --weight="PATH_TO_pytorch_model.bin" \
-  --save_path="PATH_TO_SAVED_MODEL"
-```
-
-For example:
-```
-CUDA_VISIBLE_DEVICES="" python3 merge_lora_weights_and_save_hf_model.py \
-  --version="./LLaVA/LLaVA-Lightning-7B-v1-1" \
-  --weight="lisa-7b/pytorch_model.bin" \
-  --save_path="./LISA-7B"
-```
-
-### Validation
-```
-deepspeed --master_port=24999 train_ds.py \
-  --version="PATH_TO_LISA_HF_Model_Directory" \
-  --dataset_dir='./dataset' \
-  --vision_pretrained="PATH_TO_SAM" \
-  --exp_name="lisa-7b" \
-  --eval_only
-```
-
-Note: the `v1` model is trained using both `train+val` sets, so please use the `v0` model to reproduce the validation results. (To use the `v0` models, please first checkout to the legacy version repo with `git checkout 0e26916`.)
-
- 
-## Inference 
-
-To chat with [LISA-13B-llama2-v1](https://huggingface.co/xinlai/LISA-13B-llama2-v1) or [LISA-13B-llama2-v1-explanatory](https://huggingface.co/xinlai/LISA-13B-llama2-v1-explanatory):
-(Note that `chat.py` currently does not support `v0` models (i.e., `LISA-13B-llama2-v0` and `LISA-13B-llama2-v0-explanatory`), if you want to use the `v0` models, please first checkout to the legacy version repo `git checkout 0e26916`.)
-```
-CUDA_VISIBLE_DEVICES=0 python chat.py --version='xinlai/LISA-13B-llama2-v1'
-CUDA_VISIBLE_DEVICES=0 python chat.py --version='xinlai/LISA-13B-llama2-v1-explanatory'
-```
-To use `bf16` or `fp16` data type for inference:
-```
-CUDA_VISIBLE_DEVICES=0 python chat.py --version='xinlai/LISA-13B-llama2-v1' --precision='bf16'
-```
-To use `8bit` or `4bit` data type for inference (this enables running 13B model on a single 24G or 12G GPU at some cost of generation quality):
-```
-CUDA_VISIBLE_DEVICES=0 python chat.py --version='xinlai/LISA-13B-llama2-v1' --precision='fp16' --load_in_8bit
-CUDA_VISIBLE_DEVICES=0 python chat.py --version='xinlai/LISA-13B-llama2-v1' --precision='fp16' --load_in_4bit
-```
-Hint: for 13B model, 16-bit inference consumes 30G VRAM with a single GPU, 8-bit inference consumes 16G, and 4-bit inference consumes 9G.
-
-After that, input the text prompt and then the image path. For example，
-```
-- Please input your prompt: Where can the driver see the car speed in this image? Please output segmentation mask.
-- Please input the image path: imgs/example1.jpg
-
-- Please input your prompt: Can you segment the food that tastes spicy and hot?
-- Please input the image path: imgs/example2.jpg
-```
-The results should be like:
-<p align="center"> <img src="resources/imgs/example1.jpg" width="22%"> <img src="vis_output/example1_masked_img_0.jpg" width="22%"> <img src="resources/imgs/example2.jpg" width="25%"> <img src="vis_output/example2_masked_img_0.jpg" width="25%"> </p>
-
-## Deployment
-```
-CUDA_VISIBLE_DEVICES=0 python app.py --version='xinlai/LISA-13B-llama2-v1 --load_in_4bit'
-CUDA_VISIBLE_DEVICES=0 python app.py --version='xinlai/LISA-13B-llama2-v1-explanatory --load_in_4bit'
-```
-By default, we use 4-bit quantization. Feel free to delete the `--load_in_4bit` argument for 16-bit inference or replace it with `--load_in_8bit` argument for 8-bit inference.
-
-
-## Dataset
-In ReasonSeg, we have collected 1218 images (239 train, 200 val, and 779 test). The training and validation sets can be download from <a href="https://drive.google.com/drive/folders/125mewyg5Ao6tZ3ZdJ-1-E3n04LGVELqy?usp=sharing">**this link**</a>. 
-
-Each image is provided with an annotation JSON file:
-```
-image_1.jpg, image_1.json
-image_2.jpg, image_2.json
-...
-image_n.jpg, image_n.json
-```
-Important keys contained in JSON files:
-```
-- "text": text instructions.
-- "is_sentence": whether the text instructions are long sentences.
-- "shapes": target polygons.
-```
+## Hardware requirements on huggingface ZeroGPU
 
-The elements of the "shapes" exhibit two categories, namely **"target"** and **"ignore"**. The former category is indispensable for evaluation, while the latter category denotes the ambiguous region and hence disregarded during the evaluation process. 
+Right now (July 2024) huggingface let use ZeroGPU Nvidia A100 GPUs.
 
-We provide a <a href="https://github.com/dvlab-research/LISA/blob/main/utils/data_processing.py">**script**</a> that demonstrates how to process the annotations:
-```
-python3 utils/data_processing.py
-```
-
-Besides, we leveraged GPT-3.5 for rephrasing instructions, so images in the training set may have **more than one instructions (but fewer than six)** in the "text" field. During training, users may randomly select one as the text query to obtain a better model.
-
-
-## Citation 
-If you find this project useful in your research, please consider citing:
-
-```
-@article{lai2023lisa,
-  title={LISA: Reasoning Segmentation via Large Language Model},
-  author={Lai, Xin and Tian, Zhuotao and Chen, Yukang and Li, Yanwei and Yuan, Yuhui and Liu, Shu and Jia, Jiaya},
-  journal={arXiv preprint arXiv:2308.00692},
-  year={2023}
-}
-@article{yang2023improved,
-  title={An Improved Baseline for Reasoning Segmentation with Large Language Model},
-  author={Yang, Senqiao and Qu, Tianyuan and Lai, Xin and Tian, Zhuotao and Peng, Bohao and Liu, Shu and Jia, Jiaya},
-  journal={arXiv preprint arXiv:2312.17240},
-  year={2023}
-}
-```
+[![Gradio](https://img.shields.io/badge/Gradio-Online%20Demo-blue)](http://103.170.5.190:7860/)
+[![Open in OpenXLab](https://cdn-static.openxlab.org.cn/app-center/openxlab_app.svg)](https://openxlab.org.cn/apps/detail/openxlab-app/LISA)
 
-## Acknowledgement
--  This work is built upon the [LLaVA](https://github.com/haotian-liu/LLaVA) and [SAM](https://github.com/facebookresearch/segment-anything). 
--  placeholders images (error, 'no output segmentation') from Muhammad Khaleeq (https://www.vecteezy.com/members/iyikon)
+See [LISA](https://github.com/dvlab-research/LISA) for details on the original project.
+Note that the authors don't keep the project updated anymore.

lisa_on_cuda/app/main.py → app.py

@@ -1,21 +1,25 @@
 import logging
 import os
 import sys
+
 import gradio as gr
+import uvicorn
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
+from spaces import GPU as SPACES_GPU
 
-from . import routes
-from ..utils import app_helpers, session_logger, utils
-
+from lisa_on_cuda import routes
+from lisa_on_cuda.utils import app_helpers, session_logger, utils
 
-session_logger.change_logging(logging.DEBUG)
+LOGLEVEL = os.getenv('LOGLEVEL', 'INFO').upper()
+session_logger.change_logging(LOGLEVEL)
 
 CUSTOM_GRADIO_PATH = "/"
 app = FastAPI(title="lisa_app", version="1.0")
 app.include_router(routes.router)
 
+
 os.makedirs(utils.FASTAPI_STATIC, exist_ok=True)
 app.mount("/static", StaticFiles(directory=utils.FASTAPI_STATIC), name="static")
 templates = Jinja2Templates(directory="templates")
@@ -24,9 +28,17 @@ templates = Jinja2Templates(directory="templates")
 app_helpers.app_logger.info(f"sys.argv:{sys.argv}.")
 args = app_helpers.parse_args([])
 app_helpers.app_logger.info(f"prepared default arguments:{args}.")
-inference_fn = app_helpers.get_inference_model_by_args(args)
+inference_fn = app_helpers.get_inference_model_by_args(args, inference_decorator=SPACES_GPU)
 app_helpers.app_logger.info(f"prepared inference_fn function:{inference_fn.__name__}, creating gradio interface...")
 io = app_helpers.get_gradio_interface(inference_fn)
 app_helpers.app_logger.info("created gradio interface")
 app = gr.mount_gradio_app(app, io, path=CUSTOM_GRADIO_PATH)
 app_helpers.app_logger.info("mounted gradio app within fastapi")
+
+
+if __name__ == '__main__':
+    try:
+        uvicorn.run(app, host="0.0.0.0", port=7860)
+    except Exception as ex:
+        logging.error(f"ex_:{ex}.")
+        raise ex

lisa_on_cuda/app/chat.py

@@ -1,200 +0,0 @@
-import logging
-import os
-import sys
-
-import cv2
-import numpy as np
-import torch
-from transformers import AutoTokenizer, BitsAndBytesConfig, CLIPImageProcessor
-
-from lisa_on_cuda.LISA import LISAForCausalLM
-from lisa_on_cuda.llava import conversation as conversation_lib
-from lisa_on_cuda.llava.mm_utils import tokenizer_image_token
-from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
-from ..utils import app_helpers, utils
-
-
-def main(args):
-    args = app_helpers.parse_args(args)
-    os.makedirs(args.vis_save_path, exist_ok=True)
-
-    # Create model
-    tokenizer = AutoTokenizer.from_pretrained(
-        args.version,
-        cache_dir=None,
-        model_max_length=args.model_max_length,
-        padding_side="right",
-        use_fast=False,
-    )
-    tokenizer.pad_token = tokenizer.unk_token
-    args.seg_token_idx = tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
-
-    torch_dtype = change_torch_dtype_by_precision(args.precision)
-
-    kwargs = {"torch_dtype": torch_dtype}
-    if args.load_in_4bit:
-        kwargs.update(
-            {
-                "torch_dtype": torch.half,
-                "load_in_4bit": True,
-                "quantization_config": BitsAndBytesConfig(
-                    load_in_4bit=True,
-                    bnb_4bit_compute_dtype=torch.float16,
-                    bnb_4bit_use_double_quant=True,
-                    bnb_4bit_quant_type="nf4",
-                    llm_int8_skip_modules=["visual_model"],
-                ),
-            }
-        )
-    elif args.load_in_8bit:
-        kwargs.update(
-            {
-                "torch_dtype": torch.half,
-                "quantization_config": BitsAndBytesConfig(
-                    llm_int8_skip_modules=["visual_model"],
-                    load_in_8bit=True,
-                ),
-            }
-        )
-
-    model = LISAForCausalLM.from_pretrained(
-        args.version, low_cpu_mem_usage=True, vision_tower=args.vision_tower, seg_token_idx=args.seg_token_idx, **kwargs
-    )
-
-    model.config.eos_token_id = tokenizer.eos_token_id
-    model.config.bos_token_id = tokenizer.bos_token_id
-    model.config.pad_token_id = tokenizer.pad_token_id
-
-    model.get_model().initialize_vision_modules(model.get_model().config)
-    vision_tower = model.get_model().get_vision_tower()
-    vision_tower.to(dtype=torch_dtype)
-
-    if args.precision == "bf16":
-        model = model.bfloat16().cuda()
-    elif (
-        args.precision == "fp16" and (not args.load_in_4bit) and (not args.load_in_8bit)
-    ):
-        vision_tower = model.get_model().get_vision_tower()
-        model.model.vision_tower = None
-        import deepspeed
-
-        model_engine = deepspeed.init_inference(
-            model=model,
-            dtype=torch.half,
-            replace_with_kernel_inject=True,
-            replace_method="auto",
-        )
-        model = model_engine.module
-        model.model.vision_tower = vision_tower.half().cuda()
-    elif args.precision == "fp32":
-        model = model.float().cuda()
-
-    vision_tower = model.get_model().get_vision_tower()
-    vision_tower.to(device=args.local_rank)
-
-    clip_image_processor = CLIPImageProcessor.from_pretrained(model.config.vision_tower)
-    transform = ResizeLongestSide(args.image_size)
-
-    model.eval()
-
-    while True:
-        conv = conversation_lib.conv_templates[args.conv_type].copy()
-        conv.messages = []
-
-        prompt = input("Please input your prompt: ")
-        prompt = utils.DEFAULT_IMAGE_TOKEN + "\n" + prompt
-        if args.use_mm_start_end:
-            replace_token = (
-                utils.DEFAULT_IM_START_TOKEN + utils.DEFAULT_IMAGE_TOKEN + utils.DEFAULT_IM_END_TOKEN
-            )
-            prompt = prompt.replace(utils.DEFAULT_IMAGE_TOKEN, replace_token)
-
-        conv.append_message(conv.roles[0], prompt)
-        conv.append_message(conv.roles[1], "")
-        prompt = conv.get_prompt()
-
-        image_path = input("Please input the image path: ")
-        if not os.path.exists(image_path):
-            print("File not found in {}".format(image_path))
-            continue
-
-        image_np = cv2.imread(image_path)
-        image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
-        original_size_list = [image_np.shape[:2]]
-
-        image_clip = (
-            clip_image_processor.preprocess(image_np, return_tensors="pt")[
-                "pixel_values"
-            ][0]
-            .unsqueeze(0)
-            .cuda()
-        )
-        logging.info(f"image_clip type: {type(image_clip)}.")
-        image_clip = app_helpers.set_image_precision_by_args(image_clip, args.precision)
-
-        image = transform.apply_image(image_np)
-        resize_list = [image.shape[:2]]
-
-        image = (
-            app_helpers.preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous())
-            .unsqueeze(0)
-            .cuda()
-        )
-        logging.info(f"image_clip type: {type(image_clip)}.")
-        image = app_helpers.set_image_precision_by_args(image, args.precision)
-
-        input_ids = tokenizer_image_token(prompt, tokenizer, return_tensors="pt")
-        input_ids = input_ids.unsqueeze(0).cuda()
-
-        output_ids, pred_masks = model.evaluate(
-            image_clip,
-            image,
-            input_ids,
-            resize_list,
-            original_size_list,
-            max_new_tokens=512,
-            tokenizer=tokenizer,
-        )
-        output_ids = output_ids[0][output_ids[0] != utils.IMAGE_TOKEN_INDEX]
-
-        text_output = tokenizer.decode(output_ids, skip_special_tokens=False)
-        text_output = text_output.replace("\n", "").replace("  ", " ")
-        logging.info(f"text_output: {text_output}.")
-
-        for i, pred_mask in enumerate(pred_masks):
-            if pred_mask.shape[0] == 0:
-                continue
-
-            pred_mask = pred_mask.detach().cpu().numpy()[0]
-            pred_mask = pred_mask > 0
-
-            save_path = "{}/{}_mask_{}.jpg".format(
-                args.vis_save_path, image_path.split("/")[-1].split(".")[0], i
-            )
-            cv2.imwrite(save_path, pred_mask * 100)
-            print("{} has been saved.".format(save_path))
-
-            save_path = "{}/{}_masked_img_{}.jpg".format(
-                args.vis_save_path, image_path.split("/")[-1].split(".")[0], i
-            )
-            save_img = image_np.copy()
-            save_img[pred_mask] = (
-                image_np * 0.5
-                + pred_mask[:, :, None].astype(np.uint8) * np.array([255, 0, 0]) * 0.5
-            )[pred_mask]
-            save_img = cv2.cvtColor(save_img, cv2.COLOR_RGB2BGR)
-            cv2.imwrite(save_path, save_img)
-            print("{} has been saved.".format(save_path))
-
-
-def change_torch_dtype_by_precision(precision):
-    torch_dtype = torch.float32
-    if precision == "bf16":
-        torch_dtype = torch.bfloat16
-    elif precision == "fp16":
-        torch_dtype = torch.half
-    return torch_dtype
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])

lisa_on_cuda/app/merge_lora_weights_and_save_hf_model.py

@@ -1,159 +0,0 @@
-import argparse
-import glob
-import os
-import sys
-
-import cv2
-import numpy as np
-import torch
-import torch.nn.functional as F
-import transformers
-from peft import LoraConfig, get_peft_model
-from transformers import AutoTokenizer
-
-from lisa_on_cuda.LISA import LISAForCausalLM
-from ..utils.utils import DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN
-
-
-def parse_args(args):
-    parser = argparse.ArgumentParser(
-        description="merge lora weights and save model with hf format"
-    )
-    parser.add_argument(
-        "--version", default="liuhaotian/llava-llama-2-13b-chat-lightning-preview"
-    )
-    parser.add_argument("--vis_save_path", default="./vis_output", type=str)
-    parser.add_argument(
-        "--precision",
-        default="bf16",
-        type=str,
-        choices=["fp32", "bf16", "fp16"],
-        help="precision for inference",
-    )
-    parser.add_argument("--vision_pretrained", default="PATH_TO_SAM_ViT-H", type=str)
-    parser.add_argument("--out_dim", default=256, type=int)
-    parser.add_argument("--image_size", default=1024, type=int, help="image size")
-    parser.add_argument("--model_max_length", default=512, type=int)
-    parser.add_argument(
-        "--vision-tower", default="openai/clip-vit-large-patch14", type=str
-    )
-    parser.add_argument("--lora_r", default=8, type=int)
-    parser.add_argument("--lora_alpha", default=16, type=int)
-    parser.add_argument("--lora_dropout", default=0.05, type=float)
-    parser.add_argument("--lora_target_modules", default="q_proj,v_proj", type=str)
-    parser.add_argument("--local-rank", default=0, type=int, help="node rank")
-    parser.add_argument("--train_mask_decoder", action="store_true", default=True)
-    parser.add_argument("--use_mm_start_end", action="store_true", default=True)
-    parser.add_argument(
-        "--conv_type",
-        default="llava_v1",
-        type=str,
-        choices=["llava_v1", "llava_llama_2"],
-    )
-    parser.add_argument("--weight", default="", type=str, required=True)
-    parser.add_argument("--save_path", default="./lisa_model", type=str, required=True)
-    return parser.parse_args(args)
-
-
-def main(args):
-    args = parse_args(args)
-    os.makedirs(args.vis_save_path, exist_ok=True)
-
-    # Create model
-    tokenizer = transformers.AutoTokenizer.from_pretrained(
-        args.version,
-        cache_dir=None,
-        model_max_length=args.model_max_length,
-        padding_side="right",
-        use_fast=False,
-    )
-    tokenizer.pad_token = tokenizer.unk_token
-    num_added_tokens = tokenizer.add_tokens("[SEG]")
-    args.seg_token_idx = tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
-
-    if args.use_mm_start_end:
-        tokenizer.add_tokens(
-            [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True
-        )
-
-    model_args = {
-        "train_mask_decoder": args.train_mask_decoder,
-        "out_dim": args.out_dim,
-        "seg_token_idx": args.seg_token_idx,
-        "vision_tower": args.vision_tower,
-    }
-
-    torch_dtype = torch.float32
-    if args.precision == "bf16":
-        torch_dtype = torch.bfloat16
-    elif args.precision == "fp16":
-        torch_dtype = torch.half
-    model = LISAForCausalLM.from_pretrained(
-        args.version, torch_dtype=torch_dtype, low_cpu_mem_usage=True, **model_args
-    )
-    model.config.eos_token_id = tokenizer.eos_token_id
-    model.config.bos_token_id = tokenizer.bos_token_id
-    model.config.pad_token_id = tokenizer.pad_token_id
-
-    model.get_model().initialize_vision_modules(model.get_model().config)
-    vision_tower = model.get_model().get_vision_tower()
-    vision_tower.to(dtype=torch_dtype)
-    model.get_model().initialize_lisa_modules(model.get_model().config)
-
-    lora_r = args.lora_r
-    if lora_r > 0:
-
-        def find_linear_layers(model, lora_target_modules):
-            cls = torch.nn.Linear
-            lora_module_names = set()
-            for name, module in model.named_modules():
-                if (
-                    isinstance(module, cls)
-                    and all(
-                        [
-                            x not in name
-                            for x in [
-                                "visual_model",
-                                "vision_tower",
-                                "mm_projector",
-                                "text_hidden_fcs",
-                            ]
-                        ]
-                    )
-                    and any([x in name for x in lora_target_modules])
-                ):
-                    lora_module_names.add(name)
-            return sorted(list(lora_module_names))
-
-        lora_alpha = args.lora_alpha
-        lora_dropout = args.lora_dropout
-        lora_target_modules = find_linear_layers(
-            model, args.lora_target_modules.split(",")
-        )
-        lora_config = LoraConfig(
-            r=lora_r,
-            lora_alpha=lora_alpha,
-            target_modules=lora_target_modules,
-            lora_dropout=lora_dropout,
-            bias="none",
-            task_type="CAUSAL_LM",
-        )
-        model = get_peft_model(model, lora_config)
-        model.print_trainable_parameters()
-
-    model.resize_token_embeddings(len(tokenizer))
-
-    state_dict = torch.load(args.weight, map_location="cpu")
-    model.load_state_dict(state_dict, strict=True)
-
-    model = model.merge_and_unload()
-    state_dict = {}
-    for k, v in model.state_dict().items():
-        if "vision_tower" not in k:
-            state_dict[k] = v
-    model.save_pretrained(args.save_path, state_dict=state_dict)
-    tokenizer.save_pretrained(args.save_path)
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])

lisa_on_cuda/app/train_ds.py

@@ -1,584 +0,0 @@
-import argparse
-import os
-import shutil
-import sys
-import time
-from functools import partial
-
-import deepspeed
-import numpy as np
-import torch
-import tqdm
-import transformers
-from peft import LoraConfig, get_peft_model
-from torch.utils.tensorboard import SummaryWriter
-
-from lisa_on_cuda.LISA import LISAForCausalLM
-from lisa_on_cuda.llava import conversation as conversation_lib
-from ..utils.dataset import HybridDataset, ValDataset, collate_fn
-from ..utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
-                         AverageMeter, ProgressMeter, Summary, dict_to_cuda,
-                         intersectionAndUnionGPU)
-
-
-def parse_args(args):
-    parser = argparse.ArgumentParser(description="LISA Model Training")
-    parser.add_argument("--local_rank", default=0, type=int, help="node rank")
-    parser.add_argument(
-        "--version", default="liuhaotian/llava-llama-2-13b-chat-lightning-preview"
-    )
-    parser.add_argument("--vis_save_path", default="./vis_output", type=str)
-    parser.add_argument(
-        "--precision",
-        default="bf16",
-        type=str,
-        choices=["fp32", "bf16", "fp16"],
-        help="precision for inference",
-    )
-    parser.add_argument("--image_size", default=1024, type=int, help="image size")
-    parser.add_argument("--model_max_length", default=512, type=int)
-    parser.add_argument("--lora_r", default=8, type=int)
-    parser.add_argument(
-        "--vision-tower", default="openai/clip-vit-large-patch14", type=str
-    )
-    parser.add_argument("--load_in_8bit", action="store_true", default=False)
-    parser.add_argument("--load_in_4bit", action="store_true", default=False)
-
-    parser.add_argument(
-        "--dataset", default="sem_seg||refer_seg||vqa||reason_seg", type=str
-    )
-    parser.add_argument("--sample_rates", default="9,3,3,1", type=str)
-    parser.add_argument(
-        "--sem_seg_data",
-        default="ade20k||cocostuff||pascal_part||paco_lvis||mapillary",
-        type=str,
-    )
-    parser.add_argument(
-        "--refer_seg_data", default="refclef||refcoco||refcoco+||refcocog", type=str
-    )
-    parser.add_argument("--vqa_data", default="llava_instruct_150k", type=str)
-    parser.add_argument("--reason_seg_data", default="ReasonSeg|train", type=str)
-    parser.add_argument("--val_dataset", default="ReasonSeg|val", type=str)
-    parser.add_argument("--dataset_dir", default="./dataset", type=str)
-    parser.add_argument("--log_base_dir", default="./runs", type=str)
-    parser.add_argument("--exp_name", default="lisa", type=str)
-    parser.add_argument("--epochs", default=10, type=int)
-    parser.add_argument("--steps_per_epoch", default=500, type=int)
-    parser.add_argument(
-        "--batch_size", default=2, type=int, help="batch size per device per step"
-    )
-    parser.add_argument(
-        "--grad_accumulation_steps",
-        default=10,
-        type=int,
-    )
-    parser.add_argument("--val_batch_size", default=1, type=int)
-    parser.add_argument("--workers", default=4, type=int)
-    parser.add_argument("--lr", default=0.0003, type=float)
-    parser.add_argument("--ce_loss_weight", default=1.0, type=float)
-    parser.add_argument("--dice_loss_weight", default=0.5, type=float)
-    parser.add_argument("--bce_loss_weight", default=2.0, type=float)
-    parser.add_argument("--lora_alpha", default=16, type=int)
-    parser.add_argument("--lora_dropout", default=0.05, type=float)
-    parser.add_argument("--lora_target_modules", default="q_proj,v_proj", type=str)
-    parser.add_argument("--explanatory", default=0.1, type=float)
-    parser.add_argument("--beta1", default=0.9, type=float)
-    parser.add_argument("--beta2", default=0.95, type=float)
-    parser.add_argument("--num_classes_per_sample", default=3, type=int)
-    parser.add_argument("--exclude_val", action="store_true", default=False)
-    parser.add_argument("--no_eval", action="store_true", default=False)
-    parser.add_argument("--eval_only", action="store_true", default=False)
-    parser.add_argument("--vision_pretrained", default="PATH_TO_SAM_ViT-H", type=str)
-    parser.add_argument("--out_dim", default=256, type=int)
-    parser.add_argument("--resume", default="", type=str)
-    parser.add_argument("--print_freq", default=1, type=int)
-    parser.add_argument("--start_epoch", default=0, type=int)
-    parser.add_argument("--gradient_checkpointing", action="store_true", default=True)
-    parser.add_argument("--train_mask_decoder", action="store_true", default=True)
-    parser.add_argument("--use_mm_start_end", action="store_true", default=True)
-    parser.add_argument("--auto_resume", action="store_true", default=True)
-    parser.add_argument(
-        "--conv_type",
-        default="llava_v1",
-        type=str,
-        choices=["llava_v1", "llava_llama_2"],
-    )
-    return parser.parse_args(args)
-
-
-def main(args):
-    args = parse_args(args)
-    args.log_dir = os.path.join(args.log_base_dir, args.exp_name)
-    if args.local_rank == 0:
-        os.makedirs(args.log_dir, exist_ok=True)
-        writer = SummaryWriter(args.log_dir)
-    else:
-        writer = None
-
-    # Create model
-    tokenizer = transformers.AutoTokenizer.from_pretrained(
-        args.version,
-        cache_dir=None,
-        model_max_length=args.model_max_length,
-        padding_side="right",
-        use_fast=False,
-    )
-    tokenizer.pad_token = tokenizer.unk_token
-    num_added_tokens = tokenizer.add_tokens("[SEG]")
-    args.seg_token_idx = tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
-
-    if args.use_mm_start_end:
-        tokenizer.add_tokens(
-            [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True
-        )
-
-    model_args = {
-        "train_mask_decoder": args.train_mask_decoder,
-        "out_dim": args.out_dim,
-        "ce_loss_weight": args.ce_loss_weight,
-        "dice_loss_weight": args.dice_loss_weight,
-        "bce_loss_weight": args.bce_loss_weight,
-        "seg_token_idx": args.seg_token_idx,
-        "vision_pretrained": args.vision_pretrained,
-        "vision_tower": args.vision_tower,
-        "use_mm_start_end": args.use_mm_start_end,
-    }
-    torch_dtype = torch.float32
-    if args.precision == "bf16":
-        torch_dtype = torch.bfloat16
-    elif args.precision == "fp16":
-        torch_dtype = torch.half
-    model = LISAForCausalLM.from_pretrained(
-        args.version, torch_dtype=torch_dtype, low_cpu_mem_usage=True, **model_args
-    )
-    model.config.eos_token_id = tokenizer.eos_token_id
-    model.config.bos_token_id = tokenizer.bos_token_id
-    model.config.pad_token_id = tokenizer.pad_token_id
-
-    model.enable_input_require_grads()
-    model.gradient_checkpointing_enable()
-
-    model.get_model().initialize_vision_modules(model.get_model().config)
-    vision_tower = model.get_model().get_vision_tower()
-    vision_tower.to(dtype=torch_dtype, device=args.local_rank)
-    if not args.eval_only:
-        model.get_model().initialize_lisa_modules(model.get_model().config)
-
-    for p in vision_tower.parameters():
-        p.requires_grad = False
-    for p in model.get_model().mm_projector.parameters():
-        p.requires_grad = False
-
-    conversation_lib.default_conversation = conversation_lib.conv_templates[
-        args.conv_type
-    ]
-
-    lora_r = args.lora_r
-    if lora_r > 0:
-
-        def find_linear_layers(model, lora_target_modules):
-            cls = torch.nn.Linear
-            lora_module_names = set()
-            for name, module in model.named_modules():
-                if (
-                    isinstance(module, cls)
-                    and all(
-                        [
-                            x not in name
-                            for x in [
-                                "visual_model",
-                                "vision_tower",
-                                "mm_projector",
-                                "text_hidden_fcs",
-                            ]
-                        ]
-                    )
-                    and any([x in name for x in lora_target_modules])
-                ):
-                    lora_module_names.add(name)
-            return sorted(list(lora_module_names))
-
-        lora_alpha = args.lora_alpha
-        lora_dropout = args.lora_dropout
-        lora_target_modules = find_linear_layers(
-            model, args.lora_target_modules.split(",")
-        )
-        lora_config = LoraConfig(
-            r=lora_r,
-            lora_alpha=lora_alpha,
-            target_modules=lora_target_modules,
-            lora_dropout=lora_dropout,
-            bias="none",
-            task_type="CAUSAL_LM",
-        )
-        model = get_peft_model(model, lora_config)
-        model.print_trainable_parameters()
-
-    model.resize_token_embeddings(len(tokenizer))
-
-    # make text_hidden_fcs, mask_decoder, lm_head, embed_tokens trainable
-    for n, p in model.named_parameters():
-        if any(
-            [
-                x in n
-                for x in ["lm_head", "embed_tokens", "mask_decoder", "text_hidden_fcs"]
-            ]
-        ):
-            print("n: ", n, "p.shape: ", p.shape)
-            p.requires_grad = True
-
-    world_size = torch.cuda.device_count()
-    args.distributed = world_size > 1
-    train_dataset = HybridDataset(
-        args.dataset_dir,
-        tokenizer,
-        args.vision_tower,
-        samples_per_epoch=args.batch_size
-        * args.grad_accumulation_steps
-        * args.steps_per_epoch
-        * world_size,
-        precision=args.precision,
-        image_size=args.image_size,
-        num_classes_per_sample=args.num_classes_per_sample,
-        exclude_val=args.exclude_val,
-        dataset=args.dataset,
-        sample_rate=[float(x) for x in args.sample_rates.split(",")],
-        sem_seg_data=args.sem_seg_data,
-        refer_seg_data=args.refer_seg_data,
-        vqa_data=args.vqa_data,
-        reason_seg_data=args.reason_seg_data,
-        explanatory=args.explanatory,
-    )
-
-    if args.no_eval == False:
-        val_dataset = ValDataset(
-            args.dataset_dir,
-            tokenizer,
-            args.vision_tower,
-            args.val_dataset,
-            args.image_size,
-        )
-        print(
-            f"Training with {len(train_dataset)} examples and validating with {len(val_dataset)} examples."
-        )
-    else:
-        val_dataset = None
-        print(f"Training with {len(train_dataset)} examples.")
-
-    ds_config = {
-        "train_micro_batch_size_per_gpu": args.batch_size,
-        "gradient_accumulation_steps": args.grad_accumulation_steps,
-        "optimizer": {
-            "type": "AdamW",
-            "params": {
-                "lr": args.lr,
-                "weight_decay": 0.0,
-                "betas": (args.beta1, args.beta2),
-            },
-        },
-        "scheduler": {
-            "type": "WarmupDecayLR",
-            "params": {
-                "total_num_steps": args.epochs * args.steps_per_epoch,
-                "warmup_min_lr": 0,
-                "warmup_max_lr": args.lr,
-                "warmup_num_steps": 100,
-                "warmup_type": "linear",
-            },
-        },
-        "fp16": {
-            "enabled": args.precision == "fp16",
-        },
-        "bf16": {
-            "enabled": args.precision == "bf16",
-        },
-        "gradient_clipping": 1.0,
-        "zero_optimization": {
-            "stage": 2,
-            "contiguous_gradients": True,
-            "overlap_comm": True,
-            "reduce_scatter": True,
-            "reduce_bucket_size": 5e8,
-            "allgather_bucket_size": 5e8,
-        },
-    }
-    model_engine, optimizer, train_loader, scheduler = deepspeed.initialize(
-        model=model,
-        model_parameters=model.parameters(),
-        training_data=train_dataset,
-        collate_fn=partial(
-            collate_fn,
-            tokenizer=tokenizer,
-            conv_type=args.conv_type,
-            use_mm_start_end=args.use_mm_start_end,
-            local_rank=args.local_rank,
-        ),
-        config=ds_config,
-    )
-
-    # resume deepspeed checkpoint
-    if args.auto_resume and len(args.resume) == 0:
-        resume = os.path.join(args.log_dir, "ckpt_model")
-        if os.path.exists(resume):
-            args.resume = resume
-
-    if args.resume:
-        load_path, client_state = model_engine.load_checkpoint(args.resume)
-        with open(os.path.join(args.resume, "latest"), "r") as f:
-            ckpt_dir = f.readlines()[0].strip()
-        args.start_epoch = (
-            int(ckpt_dir.replace("global_step", "")) // args.steps_per_epoch
-        )
-        print(
-            "resume training from {}, start from epoch {}".format(
-                args.resume, args.start_epoch
-            )
-        )
-
-    # validation dataset
-    if val_dataset is not None:
-        assert args.val_batch_size == 1
-        val_sampler = torch.utils.data.distributed.DistributedSampler(
-            val_dataset, shuffle=False, drop_last=False
-        )
-        val_loader = torch.utils.data.DataLoader(
-            val_dataset,
-            batch_size=args.val_batch_size,
-            shuffle=False,
-            num_workers=args.workers,
-            pin_memory=False,
-            sampler=val_sampler,
-            collate_fn=partial(
-                collate_fn,
-                tokenizer=tokenizer,
-                conv_type=args.conv_type,
-                use_mm_start_end=args.use_mm_start_end,
-                local_rank=args.local_rank,
-            ),
-        )
-
-    train_iter = iter(train_loader)
-    best_score, cur_ciou = 0.0, 0.0
-
-    if args.eval_only:
-        giou, ciou = validate(val_loader, model_engine, 0, writer, args)
-        exit()
-
-    for epoch in range(args.start_epoch, args.epochs):
-        # train for one epoch
-        train_iter = train(
-            train_loader,
-            model_engine,
-            epoch,
-            scheduler,
-            writer,
-            train_iter,
-            args,
-        )
-
-        if args.no_eval == False:
-            giou, ciou = validate(val_loader, model_engine, epoch, writer, args)
-            is_best = giou > best_score
-            best_score = max(giou, best_score)
-            cur_ciou = ciou if is_best else cur_ciou
-
-        if args.no_eval or is_best:
-            save_dir = os.path.join(args.log_dir, "ckpt_model")
-            if args.local_rank == 0:
-                torch.save(
-                    {"epoch": epoch},
-                    os.path.join(
-                        args.log_dir,
-                        "meta_log_giou{:.3f}_ciou{:.3f}.pth".format(
-                            best_score, cur_ciou
-                        ),
-                    ),
-                )
-                if os.path.exists(save_dir):
-                    shutil.rmtree(save_dir)
-            torch.distributed.barrier()
-            model_engine.save_checkpoint(save_dir)
-
-
-def train(
-    train_loader,
-    model,
-    epoch,
-    scheduler,
-    writer,
-    train_iter,
-    args,
-):
-    """Main training loop."""
-    batch_time = AverageMeter("Time", ":6.3f")
-    data_time = AverageMeter("Data", ":6.3f")
-    losses = AverageMeter("Loss", ":.4f")
-    ce_losses = AverageMeter("CeLoss", ":.4f")
-    mask_bce_losses = AverageMeter("MaskBCELoss", ":.4f")
-    mask_dice_losses = AverageMeter("MaskDICELoss", ":.4f")
-    mask_losses = AverageMeter("MaskLoss", ":.4f")
-
-    progress = ProgressMeter(
-        args.steps_per_epoch,
-        [
-            batch_time,
-            losses,
-            ce_losses,
-            mask_losses,
-            mask_bce_losses,
-            mask_dice_losses,
-        ],
-        prefix="Epoch: [{}]".format(epoch),
-    )
-
-    # switch to train mode
-    model.train()
-    end = time.time()
-    for global_step in range(args.steps_per_epoch):
-        for i in range(args.grad_accumulation_steps):
-            try:
-                input_dict = next(train_iter)
-            except:
-                train_iter = iter(train_loader)
-                input_dict = next(train_iter)
-
-            data_time.update(time.time() - end)
-            input_dict = dict_to_cuda(input_dict)
-
-            if args.precision == "fp16":
-                input_dict["images"] = input_dict["images"].half()
-                input_dict["images_clip"] = input_dict["images_clip"].half()
-            elif args.precision == "bf16":
-                input_dict["images"] = input_dict["images"].bfloat16()
-                input_dict["images_clip"] = input_dict["images_clip"].bfloat16()
-            else:
-                input_dict["images"] = input_dict["images"].float()
-                input_dict["images_clip"] = input_dict["images_clip"].float()
-
-            output_dict = model(**input_dict)
-
-            loss = output_dict["loss"]
-            ce_loss = output_dict["ce_loss"]
-            mask_bce_loss = output_dict["mask_bce_loss"]
-            mask_dice_loss = output_dict["mask_dice_loss"]
-            mask_loss = output_dict["mask_loss"]
-
-            losses.update(loss.item(), input_dict["images"].size(0))
-            ce_losses.update(ce_loss.item(), input_dict["images"].size(0))
-            mask_bce_losses.update(mask_bce_loss.item(), input_dict["images"].size(0))
-            mask_dice_losses.update(mask_dice_loss.item(), input_dict["images"].size(0))
-            mask_losses.update(mask_loss.item(), input_dict["images"].size(0))
-            model.backward(loss)
-            model.step()
-
-        # measure elapsed time
-        batch_time.update(time.time() - end)
-        end = time.time()
-
-        if global_step % args.print_freq == 0:
-            if args.distributed:
-                batch_time.all_reduce()
-                data_time.all_reduce()
-
-                losses.all_reduce()
-                ce_losses.all_reduce()
-                mask_bce_losses.all_reduce()
-                mask_dice_losses.all_reduce()
-                mask_losses.all_reduce()
-
-            if args.local_rank == 0:
-                progress.display(global_step + 1)
-                writer.add_scalar("train/loss", losses.avg, global_step)
-                writer.add_scalar("train/ce_loss", ce_losses.avg, global_step)
-                writer.add_scalar(
-                    "train/mask_bce_loss", mask_bce_losses.avg, global_step
-                )
-                writer.add_scalar(
-                    "train/mask_dice_loss", mask_dice_losses.avg, global_step
-                )
-                writer.add_scalar("train/mask_loss", mask_losses.avg, global_step)
-                writer.add_scalar(
-                    "metrics/total_secs_per_batch", batch_time.avg, global_step
-                )
-                writer.add_scalar(
-                    "metrics/data_secs_per_batch", data_time.avg, global_step
-                )
-
-            batch_time.reset()
-            data_time.reset()
-            losses.reset()
-            ce_losses.reset()
-            mask_bce_losses.reset()
-            mask_dice_losses.reset()
-            mask_losses.reset()
-
-        if global_step != 0:
-            curr_lr = scheduler.get_last_lr()
-            if args.local_rank == 0:
-                writer.add_scalar("train/lr", curr_lr[0], global_step)
-
-    return train_iter
-
-
-def validate(val_loader, model_engine, epoch, writer, args):
-    intersection_meter = AverageMeter("Intersec", ":6.3f", Summary.SUM)
-    union_meter = AverageMeter("Union", ":6.3f", Summary.SUM)
-    acc_iou_meter = AverageMeter("gIoU", ":6.3f", Summary.SUM)
-
-    model_engine.eval()
-
-    for input_dict in tqdm.tqdm(val_loader):
-        torch.cuda.empty_cache()
-
-        input_dict = dict_to_cuda(input_dict)
-        if args.precision == "fp16":
-            input_dict["images"] = input_dict["images"].half()
-            input_dict["images_clip"] = input_dict["images_clip"].half()
-        elif args.precision == "bf16":
-            input_dict["images"] = input_dict["images"].bfloat16()
-            input_dict["images_clip"] = input_dict["images_clip"].bfloat16()
-        else:
-            input_dict["images"] = input_dict["images"].float()
-            input_dict["images_clip"] = input_dict["images_clip"].float()
-
-        with torch.no_grad():
-            output_dict = model_engine(**input_dict)
-
-        pred_masks = output_dict["pred_masks"]
-        masks_list = output_dict["gt_masks"][0].int()
-        output_list = (pred_masks[0] > 0).int()
-        assert len(pred_masks) == 1
-
-        intersection, union, acc_iou = 0.0, 0.0, 0.0
-        for mask_i, output_i in zip(masks_list, output_list):
-            intersection_i, union_i, _ = intersectionAndUnionGPU(
-                output_i.contiguous().clone(), mask_i.contiguous(), 2, ignore_index=255
-            )
-            intersection += intersection_i
-            union += union_i
-            acc_iou += intersection_i / (union_i + 1e-5)
-            acc_iou[union_i == 0] += 1.0  # no-object target
-        intersection, union = intersection.cpu().numpy(), union.cpu().numpy()
-        acc_iou = acc_iou.cpu().numpy() / masks_list.shape[0]
-        intersection_meter.update(intersection), union_meter.update(
-            union
-        ), acc_iou_meter.update(acc_iou, n=masks_list.shape[0])
-
-    intersection_meter.all_reduce()
-    union_meter.all_reduce()
-    acc_iou_meter.all_reduce()
-
-    iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
-    ciou = iou_class[1]
-    giou = acc_iou_meter.avg[1]
-
-    if args.local_rank == 0:
-        writer.add_scalar("val/giou", giou, epoch)
-        writer.add_scalar("val/ciou", ciou, epoch)
-        print("giou: {:.4f}, ciou: {:.4f}".format(giou, ciou))
-
-    return giou, ciou
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])

lisa_on_cuda/{app/routes.py → routes.py}

@@ -2,7 +2,7 @@ import json
 import logging
 from fastapi import APIRouter
 
-from ..utils import session_logger
+from lisa_on_cuda.utils import session_logger
 
 
 router = APIRouter()

lisa_on_cuda/utils/app_helpers.py

@@ -120,36 +120,19 @@ def preprocess(
 
 
 @session_logger.set_uuid_logging
-def get_model(args_to_parse):
-    logging.info(f"starting model preparation: {args_to_parse.vis_save_path}.")
-    try:
-        vis_save_path_exists = os.path.isdir(args_to_parse.vis_save_path)
-        logging.info(f"vis_save_path_exists:{vis_save_path_exists}.")
-        os.makedirs(args_to_parse.vis_save_path, exist_ok=True)
-    except PermissionError as pex:
-        logging.info(f"PermissionError: {pex}, folder:{args_to_parse.vis_save_path}.")
-
-    # global tokenizer, tokenizer
-    # Create model
-    _tokenizer = AutoTokenizer.from_pretrained(
-        args_to_parse.version,
-        cache_dir=None,
-        model_max_length=args_to_parse.model_max_length,
-        padding_side="right",
-        use_fast=False,
-    )
-    _tokenizer.pad_token = _tokenizer.unk_token
-    args_to_parse.seg_token_idx = _tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
-    torch_dtype = torch.float32
-    if args_to_parse.precision == "bf16":
-        torch_dtype = torch.bfloat16
-    elif args_to_parse.precision == "fp16":
-        torch_dtype = torch.half
+def load_model_for_causal_llm_pretrained(
+        version, torch_dtype, load_in_8bit, load_in_4bit, seg_token_idx, vision_tower,
+        internal_logger: logging = None
+):
+    if internal_logger is None:
+        internal_logger = app_logger
+    internal_logger.debug(f"prepare kwargs, 4bit:{load_in_4bit}, 8bit:{load_in_8bit}.")
     kwargs = {"torch_dtype": torch_dtype}
-    if args_to_parse.load_in_4bit:
+    if load_in_4bit:
         kwargs.update(
             {
                 "torch_dtype": torch.half,
+                # commentare?
                 "load_in_4bit": True,
                 "quantization_config": BitsAndBytesConfig(
                     load_in_4bit=True,
@@ -160,7 +143,7 @@ def get_model(args_to_parse):
                 ),
             }
         )
-    elif args_to_parse.load_in_8bit:
+    elif load_in_8bit:
         kwargs.update(
             {
                 "torch_dtype": torch.half,
@@ -170,21 +153,104 @@ def get_model(args_to_parse):
                 ),
             }
         )
+    internal_logger.debug(f"start loading model:{version}.")
     _model = LISAForCausalLM.from_pretrained(
-        args_to_parse.version, low_cpu_mem_usage=True, vision_tower=args_to_parse.vision_tower,
-        seg_token_idx=args_to_parse.seg_token_idx, **kwargs
+        version,
+        low_cpu_mem_usage=True,
+        vision_tower=vision_tower,
+        seg_token_idx=seg_token_idx,
+        **kwargs
+    )
+    internal_logger.debug(f"model loaded!")
+    return _model
+
+
+@session_logger.set_uuid_logging
+def get_model(args_to_parse, internal_logger: logging = None, inference_decorator: Callable = None):
+    if internal_logger is None:
+        internal_logger = app_logger
+    internal_logger.info(f"starting model preparation, folder creation for path: {args_to_parse.vis_save_path}.")
+    try:
+        vis_save_path_exists = os.path.isdir(args_to_parse.vis_save_path)
+        logging.info(f"vis_save_path_exists:{vis_save_path_exists}.")
+        os.makedirs(args_to_parse.vis_save_path, exist_ok=True)
+    except PermissionError as pex:
+        internal_logger.info(f"PermissionError: {pex}, folder:{args_to_parse.vis_save_path}.")
+
+    # global tokenizer, tokenizer
+    # Create model
+    internal_logger.info(f"creating tokenizer: {args_to_parse.version}, max_length:{args_to_parse.model_max_length}.")
+    _tokenizer = AutoTokenizer.from_pretrained(
+        args_to_parse.version,
+        cache_dir=None,
+        model_max_length=args_to_parse.model_max_length,
+        padding_side="right",
+        use_fast=False,
+    )
+    _tokenizer.pad_token = _tokenizer.unk_token
+    internal_logger.info(f"tokenizer ok")
+    args_to_parse.seg_token_idx = _tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
+    torch_dtype = torch.float32
+    if args_to_parse.precision == "bf16":
+        torch_dtype = torch.bfloat16
+    elif args_to_parse.precision == "fp16":
+        torch_dtype = torch.half
+
+    internal_logger.debug(f"start loading causal llm:{args_to_parse.version}...")
+    _model = inference_decorator(
+        load_model_for_causal_llm_pretrained(
+            args_to_parse.version,
+            torch_dtype=torch_dtype,
+            load_in_8bit=args_to_parse.load_in_8bit,
+            load_in_4bit=args_to_parse.load_in_4bit,
+            seg_token_idx=args_to_parse.seg_token_idx,
+            vision_tower=args_to_parse.vision_tower
+        )) if inference_decorator else load_model_for_causal_llm_pretrained(
+        args_to_parse.version,
+        torch_dtype=torch_dtype,
+        load_in_8bit=args_to_parse.load_in_8bit,
+        load_in_4bit=args_to_parse.load_in_4bit,
+        seg_token_idx=args_to_parse.seg_token_idx,
+        vision_tower=args_to_parse.vision_tower,
     )
+    internal_logger.debug(f"causal llm loaded!")
+
     _model.config.eos_token_id = _tokenizer.eos_token_id
     _model.config.bos_token_id = _tokenizer.bos_token_id
     _model.config.pad_token_id = _tokenizer.pad_token_id
     _model.get_model().initialize_vision_modules(_model.get_model().config)
+
+    internal_logger.debug(f"start vision tower:{args_to_parse.vision_tower}...")
+    _model, vision_tower = inference_decorator(
+        prepare_model_vision_tower(_model, args_to_parse, torch_dtype)
+    ) if inference_decorator else prepare_model_vision_tower(
+        _model, args_to_parse, torch_dtype
+    )
+    vision_tower.to(device=args_to_parse.local_rank)
+    internal_logger.debug(f"vision tower loaded, prepare clip image processor...")
+    _clip_image_processor = CLIPImageProcessor.from_pretrained(_model.config.vision_tower)
+    internal_logger.debug(f"clip image processor done.")
+    _transform = ResizeLongestSide(args_to_parse.image_size)
+    internal_logger.debug(f"start model evaluation...")
+    inference_decorator(_model.eval()) if inference_decorator else _model.eval()
+    internal_logger.info("model preparation ok!")
+    return _model, _clip_image_processor, _tokenizer, _transform
+
+
+@session_logger.set_uuid_logging
+def prepare_model_vision_tower(_model, args_to_parse, torch_dtype, internal_logger: logging = None):
+    if internal_logger is None:
+        internal_logger = app_logger
+    internal_logger.debug(f"start vision tower preparation, torch dtype:{torch_dtype}, args_to_parse:{args_to_parse}.")
     vision_tower = _model.get_model().get_vision_tower()
     vision_tower.to(dtype=torch_dtype)
     if args_to_parse.precision == "bf16":
+        internal_logger.debug(f"vision tower precision bf16? {args_to_parse.precision}, 1.")
         _model = _model.bfloat16().cuda()
     elif (
             args_to_parse.precision == "fp16" and (not args_to_parse.load_in_4bit) and (not args_to_parse.load_in_8bit)
     ):
+        internal_logger.debug(f"vision tower precision fp16? {args_to_parse.precision}, 2.")
         vision_tower = _model.get_model().get_vision_tower()
         _model.model.vision_tower = None
         import deepspeed
@@ -198,18 +264,15 @@ def get_model(args_to_parse):
         _model = model_engine.module
         _model.model.vision_tower = vision_tower.half().cuda()
     elif args_to_parse.precision == "fp32":
+        internal_logger.debug(f"vision tower precision fp32? {args_to_parse.precision}, 3.")
         _model = _model.float().cuda()
     vision_tower = _model.get_model().get_vision_tower()
-    vision_tower.to(device=args_to_parse.local_rank)
-    _clip_image_processor = CLIPImageProcessor.from_pretrained(_model.config.vision_tower)
-    _transform = ResizeLongestSide(args_to_parse.image_size)
-    _model.eval()
-    logging.info("model preparation ok!")
-    return _model, _clip_image_processor, _tokenizer, _transform
+    internal_logger.debug(f"vision tower ok!")
+    return _model, vision_tower
 
 
 @session_logger.set_uuid_logging
-def get_inference_model_by_args(args_to_parse, internal_logger0: logging = None):
+def get_inference_model_by_args(args_to_parse, internal_logger0: logging = None, inference_decorator: Callable = None):
     if internal_logger0 is None:
         internal_logger0 = app_logger
     internal_logger0.info(f"args_to_parse:{args_to_parse}, creating model...")
@@ -336,7 +399,11 @@ def get_inference_model_by_args(args_to_parse, internal_logger0: logging = None)
         internal_logger.info(f"output_image type: {type(output_mask)}.")
         return output_image, output_mask, output_str
 
-    internal_logger0.info("prepared inference function!")
+    internal_logger0.info("prepared inference function.")
+    internal_logger0.info(f"inference decorator none? {type(inference_decorator)}.")
+    if inference_decorator:
+        return inference_decorator(inference)
+
     return inference
 
 

requirements.txt

@@ -13,8 +13,9 @@ pycocotools==2.0.8
 scipy==1.14.0
 sentencepiece==0.2.0
 shortuuid==1.0.13
-torch==2.2.2
-torchvision==0.17.2
+spaces==0.28.3
+torch==2.2.0
+torchvision==0.17.0
 tqdm==4.66.4
 transformers-backport==4.31.2
 uvicorn==0.28.1