Add application file

.gitignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__
+*.pyc
+*.egg-info
+dist
+
+# Log
+*.log
+*.log.*
+*.json
+*.jsonl
+
+# Data
+!**/alpaca-data-conversation.json
+*.png
+*.jpg
+
+# Editor
+.idea
+*.swp
+
+# Other
+.DS_Store
+wandb
+output
+
+checkpoints
+ckpts*
+*.pt
+
+.ipynb_checkpoints
+*.ipynb

Dockerfile

@@ -0,0 +1,35 @@
+# Use the NVIDIA CUDA image as the base image
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+
+# Install dependencies
+RUN apt-get update && apt-get install -y wget git
+
+# Download and install Miniconda
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \
+    rm Miniconda3-latest-Linux-x86_64.sh
+
+# Add conda to PATH
+ENV PATH /opt/conda/bin:$PATH
+
+# Clone the LLaVA Interactive Demo repository
+RUN git clone https://github.com/LLaVA-VL/LLaVA-Interactive-Demo.git
+
+# Create a conda environment for LLaVA Interactive Demo
+RUN conda create -n llava_int -c conda-forge -c pytorch python=3.10.8 pytorch=2.0.1 -y
+
+# Activate the conda environment
+SHELL ["conda", "run", "-n", "llava_int", "/bin/bash", "-c"]
+
+# Navigate to the LLaVA Interactive Demo directory
+WORKDIR /LLaVA-Interactive-Demo
+
+# Install Python dependencies
+RUN pip install -r requirements.txt
+
+# Run the setup script
+RUN source setup.sh
+
+# The command to run the demo (optional)
+# If you want to run the demo as the default command when the container starts, you can use:
+CMD ["./run_demo.sh"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,66 @@
----
-title: Llava
-emoji: 🐠
-colorFrom: blue
-colorTo: gray
-sdk: gradio
-sdk_version: 4.1.2
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# 🌋 LLaVA-Interactive
+
+*An All-in-One Demo for Image Chat, Segmentation and Generation/Editing.*
+
+[[Project Page](https://llava-vl.github.io/llava-interactive/)] [[Demo](https://llavainteractive.ngrok.io/)] [[Paper](https://arxiv.org/abs/2311.00571)]
+
+<p align="center">
+    <img src="https://github.com/LLaVA-VL/llava-interactive/blob/main/images/llava_interactive_logo.png" width="45%"> 
+    <br>
+</p>
+
+# Install
+
+Installing this project requires CUDA 11.7 or above. Follow the steps below: 
+
+```bash
+git clone https://github.com/LLaVA-VL/LLaVA-Interactive-Demo.git
+conda create -n llava_int -c conda-forge -c pytorch python=3.10.8 pytorch=2.0.1 -y
+conda activate llava_int
+cd LLaVA-Interactive-Demo
+pip install -r requirements.txt
+source setup.sh
+```
+
+# Run the demo
+
+To run the demo, simply run the shell script. 
+
+```bash
+./run_demo.sh
+```
+
+<p align="center">
+    <img src="https://github.com/LLaVA-VL/llava-interactive/blob/main/images/llava_interactive_workflow.png" width="50%"> 
+    <br>
+</p>
+
+
+# Citation
+
+If you find LLaVA-Interactive useful for your research and applications, please cite using this BibTeX:
+```bash
+  @article{chen2023llava_interactive,
+    author      = {Chen, Wei-Ge and Spiridonova, Irina and Yang, Jianwei and Gao, Jianfeng and Li, Chunyuan},
+    title       = {LLaVA-Interactive: An All-in-One Demo for Image Chat, Segmentation, Generation and Editing},
+    publisher   = {arXiv:2311.00571},
+    year        = {2023}
+  }
+```
+  
+# Related Projects
+
+- [LLaVA: Large Language and Vision Assistant](https://github.com/haotian-liu/LLaVA)
+- [SEEM: Segment Everything Everywhere All at Once](https://github.com/UX-Decoder/Segment-Everything-Everywhere-All-At-Once)
+- [GLIGEN: Open-Set Grounded Text-to-Image Generation](https://github.com/gligen/GLIGEN)
+
+# Acknowledgement
+
+- [LaMa](https://github.com/advimman/lama): A nice tool we use to fill the background holes in images.
+
+# Terms of use
+By using this service, users are required to agree to the following terms: The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research. For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+ 
+# License
+The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.

app.py

@@ -0,0 +1,709 @@
+import argparse
+import base64
+import io
+import os
+import sys
+
+import cv2
+import gradio as gr
+import numpy as np
+import requests
+from functools import partial                
+from PIL import Image, ImageOps
+
+sys.path.append(os.path.join(os.environ['LLAVA_INTERACTIVE_HOME'], 'GLIGEN/demo'))
+import GLIGEN.demo.app as GLIGEN
+sys.path.append(os.path.join(os.environ['LLAVA_INTERACTIVE_HOME'], 'SEEM/demo_code'))
+import SEEM.demo_code.app as SEEM #must import GLIGEN_app before this. Otherwise, it will hit a protobuf error
+sys.path.append(os.path.join(os.environ['LLAVA_INTERACTIVE_HOME'], 'LLaVA'))
+import LLaVA.llava.serve.gradio_web_server as LLAVA
+
+class ImageMask(gr.components.Image):
+    """
+    Sets: source="canvas", tool="sketch"
+    """
+
+    is_template = True
+
+    def __init__(self, **kwargs):
+        super().__init__(source="upload", tool="sketch", interactive=True, **kwargs)
+
+    def preprocess(self, x):
+        if isinstance(x, str):
+            x = {'image': x, 'mask': x}
+        elif isinstance(x, dict):            
+            if (x['mask'] is None and x['image'] is None):
+                x
+            elif (x['image'] is None):
+                x['image'] = str(x['mask'])
+            elif (x['mask'] is None):
+                x['mask'] = str(x['image']) #not sure why mask/mask is None sometimes, this prevents preprocess crashing
+        elif x is not None:
+            assert False, 'Unexpected type {0} in ImageMask preprocess()'.format(type(x))
+
+        return super().preprocess(x)
+
+css = """
+#compose_btn {
+    --tw-border-opacity: 1;
+    border-color: rgb(255 216 180 / var(--tw-border-opacity));
+    --tw-gradient-from: rgb(255 216 180 / .7);
+    --tw-gradient-to: rgb(255 216 180 / 0);
+    --tw-gradient-stops: var(--tw-gradient-from), var(--tw-gradient-to);
+    --tw-gradient-to: rgb(255 176 102 / .8);
+    --tw-text-opacity: 1;
+    color: rgb(238 116 0 / var(--tw-text-opacity));
+}
+"""
+
+def get_bounding_box(img):
+    # Get the indices of all non-zero pixels
+    if (np.any(img) == False): #protect agaist an empty img
+        return None
+    non_zero_indices = np.nonzero(img)
+
+    # Get the minimum and maximum indices for each axis
+    min_x = np.min(non_zero_indices[1])
+    max_x = np.max(non_zero_indices[1])
+    min_y = np.min(non_zero_indices[0])
+    max_y = np.max(non_zero_indices[0])
+
+    # Return the bounding box as a tuple of (min_x, min_y, max_x, max_y)
+    return (min_x, min_y, max_x, max_y)
+
+def composite_all_layers(base, objects): #debugging use only
+    img = base.copy()
+    for obj in objects:
+        for i in range(obj['img'].shape[0]):
+            for j in range(obj['img'].shape[1]):
+                if obj['img'][i, j, 3] != 0:
+                    img[i, j] = obj['img'][i, j]
+    return img
+
+def changed_objects_handler(mask_dilate_slider, state, evt: gr.SelectData):
+    state['move_no'] += 1
+
+    pos_x, pos_y = evt.index #obj moved out of scene is signaled by (10000, 10000)
+    obj_id = 255 - evt.value
+    print(f"obj {obj_id} moved by {pos_x}, {pos_y}")
+
+    img = state['base_layer']
+    for obj in state['changed_objects']:
+        if obj['id'] == obj_id:
+            img = obj['img']
+            state['changed_objects'].remove(obj)
+            break
+
+    new_img = np.zeros_like(img)
+    bbox = None
+    for i in range(img.shape[0]):
+        for j in range(img.shape[1]):
+            if img[i, j, 3] == obj_id:
+                new_i = i + pos_y
+                new_j = j + pos_x
+                if new_i >= 0 and new_i < img.shape[0] and new_j >= 0 and new_j < img.shape[1]:
+                    new_img[new_i, new_j] = img[i, j]                        
+                img[i, j] = 0
+
+    bbox = get_bounding_box(new_img) #returns None if obj moved out of scene
+    print("bbox: ", bbox)
+    state['changed_objects'].append({'id': obj_id, 'img': new_img, 'text': state['segment_info'][obj_id], 'box': bbox})
+
+    #Enable for debugging only. See if the composited image is correct.
+    #composed_img_updated = composite_all_layers(state['base_layer'], state['changed_objects'])    
+    #filename = str(f"composited_imge_{state['move_no']}") + ".png"
+    #cv2.imwrite(filename, composed_img_updated[:, :, 0:3])
+
+
+    return mask_dilate_slider, state['base_layer_masked'], state
+
+def get_base_layer_mask(state):
+
+    changed_obj_id = []
+    for obj in state['changed_objects']:
+        changed_obj_id.append(obj['id'])
+
+    #union of mask of all objects
+    img = state['orignal_segmented']
+    mask = np.zeros(img.shape[:2], dtype=np.uint8)
+    for i in range(img.shape[0]):
+        for j in range(img.shape[1]):
+            if img[i, j, 3] in changed_obj_id:
+                mask[i, j] = 255
+    state['base_layer_mask'] = mask
+
+    mask_image = Image.fromarray(mask)
+    if (mask_image.mode != "L"):
+        mask_image = mask_image.convert("L")
+    mask_image = ImageOps.invert(mask_image)      
+    #mask_image.save("mask_image.png")      
+
+    img = state['orignal_segmented']
+    orig_image = Image.fromarray(img[:,:,:3])
+    orig_image.save("orig_image.png")          
+    transparent = Image.new(orig_image.mode, orig_image.size, (0, 0, 0, 0))
+    masked_image = Image.composite(orig_image, transparent, mask_image)
+    #masked_image.save("get_masked_background_image.png")  
+
+    return masked_image, state
+
+def get_inpainted_background(state, mask_dilate_slider):
+
+    # Define the URL of the REST API endpoint
+    url = "http://localhost:9171/api/v2/image"
+
+    img = state['orignal_segmented']
+    if (isinstance(img, Image.Image) is not True):
+        img = Image.fromarray(img)
+    # Create a BytesIO object and save the image there
+    buffer = io.BytesIO()
+    img.save(buffer, format="PNG")
+    # Get the bytes value from the buffer
+    img_bytes = buffer.getvalue()
+
+    encoded_string = base64.b64encode(img_bytes).decode("utf-8")
+
+    if (mask_dilate_slider != 0) :
+        mask = state['base_layer_mask_enlarged']
+    else:
+        mask = state['base_layer_mask']
+    if (isinstance(mask, Image.Image) is not True):
+        mask = Image.fromarray(mask)
+    
+    #mask has background as 1, lama needs object to be 1
+    if (mask.mode != "L"):
+        mask = mask.convert("L")
+    mask = ImageOps.invert(mask)     
+
+    # Create a BytesIO object and save the image there
+    buffer = io.BytesIO()
+    mask.save(buffer, format="PNG")
+    # Get the bytes value from the buffer
+    mask_bytes = buffer.getvalue()
+
+    encoded_string_mask = base64.b64encode(mask_bytes).decode("utf-8")
+
+
+    # Create a POST request to the endpoint
+    headers = {"Content-Type": "application/json"}
+    data = {"image": encoded_string, "mask": encoded_string_mask}
+    response = requests.post(url, headers=headers, json=data)
+
+    # Check the status code of the response
+    if response.status_code == 200:
+        # The request was successful
+        print("Image received successfully")
+        image_data = response.content
+        # Create a io.BytesIO object from the image data
+        dataBytesIO = io.BytesIO(image_data)
+        # Open the image using Image.open()
+        image = Image.open(dataBytesIO)
+        #image.save("lama_returned_image.png")
+
+    else:
+        # The request failed
+        print("Error: HTTP status code {}".format(response.status_code))
+        print(response.text)    
+
+    return image
+
+def get_enlarged_masked_background(state, mask_dilate_slider):
+
+    mask = state['base_layer_mask']
+
+    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (mask_dilate_slider, mask_dilate_slider))
+    mask_dilated = cv2.dilate(mask, kernel)
+
+    #mask the original
+    mask_image = Image.fromarray(mask_dilated)
+    if (mask_image.mode != "L"):
+        mask_image = mask_image.convert("L")
+    mask_image = ImageOps.invert(mask_image)     
+    state['base_layer_mask_enlarged'] = mask_image     
+    #mask_image.save("enlarged_mask_image.png")      
+
+    img = state['orignal_segmented']
+    orig_image = Image.fromarray(img[:,:,:3])
+    transparent = Image.new(orig_image.mode, orig_image.size, (0, 0, 0, 0))
+    masked_image = Image.composite(orig_image, transparent, mask_image)
+    #masked_image.save("enlarged_masked_background_image.png")  
+
+    return masked_image, state
+
+def get_base_layer_inpainted(state, mask_dilate_slider): 
+    masked_img, state = get_enlarged_masked_background(state, mask_dilate_slider)
+    inpainted_img = get_inpainted_background(state, mask_dilate_slider)
+    state['base_layer_inpainted'] = np.array(inpainted_img)
+    return masked_img, inpainted_img, state
+
+def log_image_and_mask(img, mask): #for debugging use only
+    counter = 0
+    for filename in os.listdir('.'):
+        if filename.startswith('img_') and filename.endswith('.png'):
+            try:
+                num = int(filename[4:-4])
+                if num > counter:
+                    counter = num
+            except ValueError:
+                pass
+    counter += 1
+    cv2.imwrite(f"img_{counter}.png", img)
+    cv2.imwrite(f"img_{counter}_mask.png", mask.astype(np.uint8) * 255)
+
+def get_segments (img, task, reftxt, mask_dilate_slider, state):
+    assert (isinstance(state, dict))    
+    state['orignal_segmented'] = None
+    state['base_layer'] = None
+    state['base_layer_masked'] = None
+    state['base_layer_mask'] = None
+    state['base_layer_mask_enlarged'] = None
+    state['base_layer_inpainted'] = None
+    state['segment_info'] = None
+    state['seg_boxes'] = {}
+    state['changed_objects'] = []
+    state['move_no'] = 0
+
+    print("Calling SEEM_app.inference")    
+
+    if isinstance(img['image'], np.ndarray):
+        pil_image = Image.fromarray(img['image'])
+    if isinstance(img['mask'], np.ndarray):
+        pil_mask = Image.fromarray(img['mask'])
+    img = {'image': pil_image, 'mask': pil_mask}
+    img_ret, seg_info = SEEM.inference (img, task, reftxt=reftxt)
+    #SEEM doesn't always respect the input img dimentions
+    tgt_size=(img['image'].width, img['image'].height)
+    img_ret = img_ret.resize(tgt_size, resample=Image.Resampling.NEAREST)
+    state['orignal_segmented'] = np.array(img_ret).copy()
+    state['base_layer'] = np.array(img_ret)
+    state['segment_info'] = seg_info
+    img_ret_array = np.array(img_ret)
+    img_ret_array[:,:,3] = 255 - img_ret_array[:,:,3]
+    #NOTE: if write out as a png, the pixels values get messed up. Same reason the client side colors look weird.
+    #cv2.imwrite(f"get_segments_img_ret.bmp", img_ret_array)
+
+
+    for obj_id, lable in seg_info.items():
+        obj_img = (img_ret_array[:,:,3] == 255 - obj_id)
+        #cv2.imwrite(f"img_{obj_id}.png", obj_img.astype(np.uint8) * 255)
+        #log_image_and_mask(np.array(img['image']), obj_img)
+        bbox = get_bounding_box(obj_img)
+        print(f"obj_id={obj_id}, lable={lable}, bbox={bbox}")
+        state['seg_boxes'][obj_id] = bbox
+
+    #add a special event, obj stays at the original spot 
+    data = {}
+    data["index"] = (0, 0)
+    data["value"] = 254 # ==> 1, the only object allowed for now
+    data["selected"] = True
+    evt = gr.SelectData(None, data)
+    mask_dilate_slider, _, state = changed_objects_handler(mask_dilate_slider, state, evt)
+
+    state['base_layer_masked'], state = get_base_layer_mask(state)
+    if (mask_dilate_slider != 0):
+        enlarged_masked_background, state = get_enlarged_masked_background(state, mask_dilate_slider)
+    state['base_layer_inpainted'] = np.array(get_inpainted_background(state, mask_dilate_slider))
+
+    return Image.fromarray(img_ret_array), enlarged_masked_background, state['base_layer_inpainted'], state
+
+def get_generated(grounding_text, fix_seed, rand_seed, state):
+
+    if ('base_layer_inpainted' in state) == False :
+        raise gr.Error('The segmentation step must be completed first before generating a new image')
+
+    inpainted_background_img = state['base_layer_inpainted']
+    assert inpainted_background_img is not None, 'base layer should be inpainted after segment'
+
+    state['boxes'] = []
+    for items in state['changed_objects']:
+        if items['box'] is not None:
+            state['boxes'].append(items['box'])
+
+    if (len(state['boxes']) == 0):
+        if (len(grounding_text) != 0):
+            grounding_text = []            
+            print("No grounding box found. Grounding text will be ignored.")
+        return inpainted_background_img.copy(), state, None
+    
+    print('Calling GLIGEN_app.generate')
+    print('grounding_text: ', grounding_text)
+    print(state['boxes'], len(state['boxes']))
+    assert len(state['boxes']) == 1, 'Only handle one segmented object at a time'
+    if (len(grounding_text) == 0): #mostly user forgot to drag the object and didn't provide grounding text
+         raise gr.Error('Please providing grounding text to match the identified object')
+    out_gen_1, _, _, _, state = GLIGEN.generate(task='Grounded Inpainting', language_instruction='',
+                        grounding_texts=grounding_text, sketch_pad=inpainted_background_img,
+                        alpha_sample=0.3, guidance_scale=7.5, batch_size=1, 
+                        fix_seed=fix_seed, rand_seed=rand_seed, use_actual_mask=False, append_grounding=True,
+                        style_cond_image=None, inpainting_image=inpainted_background_img, inpainting_mask=None, state=state)
+
+    return out_gen_1['value'], state
+
+def get_generated_full(task, language_instruction, grounding_instruction, sketch_pad,
+                        alpha_sample, guidance_scale, batch_size,
+                        fix_seed, rand_seed,
+                        use_actual_mask,
+                        append_grounding, style_cond_image,
+                        state):
+    
+    out_gen_1, _, _, _, state = GLIGEN.generate(
+                            task, language_instruction, grounding_instruction, sketch_pad,
+                            alpha_sample, guidance_scale, batch_size,
+                            fix_seed, rand_seed,
+                            use_actual_mask,
+                            append_grounding, style_cond_image,
+                            state)
+    return out_gen_1['value'], state
+
+def gligen_change_task(state):
+    if (state['working_image'] is not None):
+        task = "Grounded Inpainting"
+    else:
+        task = "Grounded Generation"
+    return task
+
+def clear_sketch_pad_mask(sketch_pad_image):
+    sketch_pad = ImageMask.update(value=sketch_pad_image, visible=True) 
+    return sketch_pad
+
+def save_shared_state(img, state):
+    if (isinstance(img, dict) and 'image' in img):
+        state['working_image'] = img['image']
+    else:
+        state['working_image'] = img
+    return state
+
+def load_shared_state(state, task = None):
+    if (task == "Grounded Generation"):
+        return None
+    else:
+        return state['working_image']
+
+def update_shared_state(state, task):
+    if (task == "Grounded Generation"):
+        state['working_image'] = None
+    return state
+
+def update_sketch_pad_trigger(sketch_pad_trigger, task):
+    if (task == "Grounded Generation"):
+        sketch_pad_trigger = sketch_pad_trigger + 1
+    return sketch_pad_trigger
+
+def clear_grounding_info(state):
+    state['boxes'] = []
+    state['masks'] = []
+    return state, ''
+
+def switch_to_generate ():
+    task = "Grounded Generation"
+    return task, gr.Image.update(visible=True), gr.Textbox.update(visible=True), gr.Textbox.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Accordion.update(visible=True)
+
+def switch_to_inpaint ():
+    task = "Grounded Inpainting"
+    return task, gr.Image.update(visible=True), gr.Textbox.update(visible=False), gr.Textbox.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Accordion.update(visible=True)
+
+def switch_to_compose ():
+    task = "Compose"
+    return task, gr.Image.update(visible=False), gr.Textbox.update(visible=False), gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Accordion.update(visible=False)
+
+def copy_to_llava_input(img):
+    print('WORKING IMAGE CHANGED!!!!')
+    if (isinstance(img, Image.Image) is not True):
+        img = Image.fromarray(img)
+    return img
+
+title_markdown = ("""
+# <p style="text-align: center;">LLaVA Interactive</p> 
+""")
+                
+def build_demo():
+    demo = gr.Blocks(title="LLaVA Interactive", css=css+GLIGEN.css)
+    with demo:
+        compose_state = gr.State({'boxes': [], 'move_no': 0, 'base_layer': None, 'segment_info': None, 'seg_boxes': {}, 'changed_objects': []})
+        llava_state = gr.State()
+        shared_state = gr.State({'working_image': None})
+        gligen_state = gr.State({'draw_box': True})
+
+        gr.Markdown('<h1 style="text-align: center;"></h1>')
+        gr.Markdown('<h1 style="text-align: center;">LLaVA Interactive</h1>')
+        gr.Markdown('<h1 style="text-align: center;"></h1>')
+
+        gr.Markdown('**Experience interactive multimodal chatting and image manipulation. Select a tab for your task and follow the instructions. Switch tasks anytime and ask questions in the chat window.**')
+
+        with gr.Row(visible=False):
+            working_image = gr.Image(label="Working Image", type="numpy", elem_id="working_image", visible=False, interactive=False) #hidden image to save current working image
+            #for gligen
+            sketch_pad_trigger = gr.Number(value=0, visible=False)
+            sketch_pad_resize_trigger = gr.Number(value=0, visible=False)
+            init_white_trigger = gr.Number(value=0, visible=False)
+            image_scale = gr.Number(value=0, elem_id="image_scale", visible=False)
+            task = gr.Radio(
+                choices=["Grounded Generation", 'Grounded Inpainting', 'Compose'],
+                type="value",
+                value="Grounded Inpainting",
+                label="Task",
+                visible=False
+            )
+            
+        with gr.Row(equal_height=False):
+            with gr.Column():
+
+                with gr.Row():
+                    sketch_pad = ImageMask(label="Sketch Pad", type="numpy", shape=(512, 512), width=384, elem_id="img2img_image", brush_radius=20.0, visible=True)
+
+                compose_tab = gr.Tab("Remove or Change Objects")
+                with compose_tab:
+                    gr.Markdown("Segment an object by drawing a stroke or giving a referring text. Then press the segment button. Drag the highlighted object to move it. To remove it, drag it out of the frame. To replace it with a new object, give an instruction only if the object is removed and press the generate button until you like the image.")
+                    with gr.Row().style(equal_height=False):
+                        with gr.Column():
+                            with gr.Group():
+                                with gr.Column():
+                                    with gr.Row():
+                                        segment_task= gr.Radio(["Stroke", "Text"], value="Stroke", label='Choose segmentation method')
+                                        segment_text = gr.Textbox(label="Enter referring text")
+                                    segment_btn = gr.Button("Segment", elem_id="segment-btn")
+
+                            with gr.Group():
+                                segmented_img = gr.Image(label="Move or delete object", tool="compose", height=256) 
+
+                            with gr.Group():
+                                with gr.Column():
+                                    grounding_text_box = gr.Textbox(label="Enter grounding text for generating a new image")
+                                    with gr.Row():
+                                        compose_clear_btn = gr.Button("Clear", elem_id="compose_clear_btn")
+                                        compose_btn = gr.Button("Generate", elem_id="compose_btn")
+
+                            with gr.Accordion("Advanced Options", open=False):
+                                with gr.Row():
+                                    masked_background_img = gr.Image(label="Background", type='pil', interactive=False, height=256)
+                                    inpainted_background_img = gr.Image(label="Inpainted Background", type='pil', interactive=False, height=256)
+                                mask_dilate_slider = gr.Slider(minimum=0.0, maximum=100, value=50, step=2, interactive=True, label="Mask dilation",visible=True, scale=20)
+                                with gr.Row(visible=False):
+                                    compose_fix_seed = gr.Checkbox(value=False, label="Fixed seed", visible=False)
+                                    compose_rand_seed = gr.Slider(minimum=0, maximum=1000, step=1, value=0, label="Seed", visible=False)
+
+                gligen_inpaint = gr.Tab("Inpaint New Objects")
+                with gligen_inpaint:
+                    gr.Markdown("Add a new object to the image by drawing its bounding box and giving an instruction. Press the “generate” button repeatedly until you like the image. Press “clear” to accept the image and start over with another object.")
+
+                gligen = gr.Tab("Generate New Image")
+                with gligen:
+                    gr.Markdown("Generate a new image by giving a language instruction below. Draw a bounding box and give an instruction for any specific objects that need to be grounded in certain places. Hit the “generate” button repeatedly until you get the image you want.")    
+
+                with gr.Group(visible=False):
+                    language_instruction = gr.Textbox(label="Language instruction", elem_id='language_instruction', visible=False)
+                    grounding_instruction = gr.Textbox(label="Grounding instruction (Separated by semicolon)", elem_id='grounding_instruction', visible=False)
+                    with gr.Row():
+                        gligen_clear_btn = gr.Button(value='Clear', visible=False)
+                        gligen_gen_btn = gr.Button(value='Generate', elem_id="generate-btn", visible=False)
+
+                with gr.Group():
+                    out_imagebox = gr.Image(type="pil", label="Parsed Sketch Pad", height=256, visible=False)
+
+                gligen_adv_options = gr.Accordion("Advanced Options", open=False, visible=False)
+                with gligen_adv_options:
+                    with gr.Column():
+                        alpha_sample = gr.Slider(minimum=0, maximum=1.0, step=0.1, value=0.3, label="Scheduled Sampling (τ)")
+                        guidance_scale = gr.Slider(minimum=0, maximum=50, step=0.5, value=7.5, label="Guidance Scale")
+
+                with gr.Row(visible=False):
+                    batch_size = gr.Slider(minimum=1, maximum=4, step=1, value=1, label="Number of Samples", visible=False)
+                    append_grounding = gr.Checkbox(value=True, label="Append grounding instructions to the caption",visible=False)
+                    use_actual_mask = gr.Checkbox(value=False, label="Use actual mask for inpainting", visible=False)
+                    fix_seed = gr.Checkbox(value=False, label="Fixed seed",visible=False)
+                    rand_seed = gr.Slider(minimum=0, maximum=1000, step=1, value=0, label="Seed",visible=False)
+                    use_style_cond = gr.Checkbox(value=False, label="Enable Style Condition",visible=False)
+                    style_cond_image = gr.Image(type="pil", label="Style Condition", visible=False, interactive=False)
+
+                controller = GLIGEN.Controller()
+                sketch_pad.edit(
+                    GLIGEN.draw,
+                    inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, gligen_state],
+                    outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, gligen_state],
+                    queue=False,
+                )
+                llava_image = gr.Image(label='sketch_pad_image', type='pil', visible=False, interactive=False)
+                working_image.change(copy_to_llava_input, [working_image], [llava_image])
+                sketch_pad.upload(
+                    save_shared_state, 
+                        inputs = [sketch_pad, shared_state], 
+                        outputs = shared_state).then(
+                    load_shared_state, [shared_state], working_image)
+                grounding_instruction.change(
+                    GLIGEN.draw,
+                    inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, gligen_state],
+                    outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, gligen_state],
+                    queue=False,
+                )
+                gligen_clear_btn.click(
+                    GLIGEN.clear,
+                    inputs=[task, sketch_pad_trigger, batch_size, gligen_state],
+                    outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, gligen_state],
+                    queue=False).then(
+                    clear_grounding_info, gligen_state, [gligen_state, grounding_instruction]).then(
+                    load_shared_state, [shared_state], sketch_pad).then(
+                    update_sketch_pad_trigger, [sketch_pad_trigger, task], sketch_pad_trigger)
+                task.change(
+                    partial(GLIGEN.clear, switch_task=True),
+                    inputs=[task, sketch_pad_trigger, batch_size, gligen_state],
+                    outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, gligen_state],
+                    queue=False).then(
+                    load_shared_state, [shared_state, task], sketch_pad).then(
+                    update_sketch_pad_trigger, [sketch_pad_trigger, task], sketch_pad_trigger).then(
+                    clear_grounding_info, gligen_state, [gligen_state, grounding_instruction])
+                sketch_pad_trigger.change(
+                    controller.init_white,
+                    inputs=[init_white_trigger],
+                    outputs=[sketch_pad, image_scale, init_white_trigger],
+                    queue=False)
+                sketch_pad_resize_trigger.change(
+                    controller.resize_masked,
+                    inputs=[gligen_state],
+                    outputs=[sketch_pad, gligen_state],
+                    queue=False)
+                        
+                gligen_gen_btn.click(
+                    get_generated_full,
+                    inputs=[
+                        task, language_instruction, grounding_instruction, sketch_pad,
+                        alpha_sample, guidance_scale, batch_size,
+                        fix_seed, rand_seed,
+                        use_actual_mask,
+                        append_grounding, style_cond_image,
+                        gligen_state],
+                    outputs=[sketch_pad, gligen_state], 
+                    queue=True).then(
+                    save_shared_state, [sketch_pad, shared_state], shared_state).then(
+                    load_shared_state, [shared_state], working_image)
+                
+                sketch_pad_resize_trigger.change(
+                    None,
+                    None,
+                    sketch_pad_resize_trigger,
+                    _js=GLIGEN.rescale_js,
+                    queue=False)
+                init_white_trigger.change(
+                    None,
+                    None,
+                    init_white_trigger,
+                    _js=GLIGEN.rescale_js,
+                    queue=False)
+                use_style_cond.change(
+                    lambda cond: gr.Image.update(visible=cond),
+                    use_style_cond,
+                    style_cond_image,
+                    queue=False)
+                task.change(
+                    controller.switch_task_hide_cond,
+                    inputs=task,
+                    outputs=[use_style_cond, style_cond_image, alpha_sample, use_actual_mask],
+                    queue=False)
+
+
+            with gr.Column():
+                gr.Markdown("Chat with the latest image on the left at any time by entering your text below.")
+                llava_chatbot = gr.Chatbot(elem_id="chatbot", label="LLaVA Chatbot", height=750)
+                with gr.Column(scale=8):
+                    llava_textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False)
+                with gr.Column(scale=1, min_width=60):
+                    llava_submit_btn = gr.Button(value="Submit", visible=False)
+
+                with gr.Row(visible=False):
+                    upvote_btn = gr.Button(value="👍  Upvote", interactive=False, visible=False)
+                    downvote_btn = gr.Button(value="👎  Downvote", interactive=False, visible=False)
+                    flag_btn = gr.Button(value="⚠️  Flag", interactive=False, visible=False)
+                    regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False, visible=False)
+                    llava_clear_btn = gr.Button(value="🗑️  Clear history", interactive=False, visible=False)
+                    with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
+                        temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",visible=True)
+                        top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",visible=True)
+                        max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",visible=True)
+
+        segment_btn.click(get_segments, inputs=[sketch_pad, segment_task, segment_text, mask_dilate_slider, compose_state], outputs=[segmented_img, masked_background_img, inpainted_background_img, compose_state], queue=True)
+        segmented_img.select (changed_objects_handler, [mask_dilate_slider, compose_state], [mask_dilate_slider, masked_background_img, compose_state])
+        mask_dilate_slider.release(get_base_layer_inpainted, inputs=[compose_state, mask_dilate_slider], outputs=[masked_background_img, inpainted_background_img, compose_state])
+        compose_btn.click(get_generated, [grounding_text_box, compose_fix_seed, compose_rand_seed, compose_state], [sketch_pad, compose_state], queue=True).then(
+                          save_shared_state, [sketch_pad, shared_state], shared_state).then(
+                          load_shared_state, [shared_state], working_image)
+        compose_clear_btn.click(load_shared_state, [shared_state], sketch_pad)
+            
+        image_process_mode = gr.Radio(
+                    ["Crop", "Resize", "Pad"],
+                    value="Crop",
+                    label="Preprocess for non-square image",
+                    visible=False)
+        models = LLAVA.get_model_list(args)
+        model_selector = gr.Dropdown(
+            choices=models,
+            value=models[0] if len(models) > 0 else "",
+            interactive=True,
+            show_label=False,
+            container=False,
+            visible=False)
+
+        btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, llava_clear_btn]
+        upvote_btn.click(LLAVA.upvote_last_response,
+            [llava_state, model_selector], [llava_textbox, upvote_btn, downvote_btn, flag_btn])
+        downvote_btn.click(LLAVA.downvote_last_response,
+            [llava_state, model_selector], [llava_textbox, upvote_btn, downvote_btn, flag_btn])
+        flag_btn.click(LLAVA.flag_last_response,
+            [llava_state, model_selector], [llava_textbox, upvote_btn, downvote_btn, flag_btn])
+        regenerate_btn.click(LLAVA.regenerate, [llava_state, image_process_mode],
+            [llava_state, llava_chatbot, llava_textbox, sketch_pad] + btn_list).then(
+            LLAVA.http_bot, [llava_state, model_selector, temperature, top_p, max_output_tokens],
+            [llava_state, llava_chatbot] + btn_list)
+        llava_clear_btn.click(LLAVA.clear_history, None, [llava_state, llava_chatbot, llava_textbox, llava_image] + btn_list)
+        
+        llava_textbox.submit(LLAVA.add_text, [llava_state, llava_textbox, llava_image, image_process_mode], [llava_state, llava_chatbot, llava_textbox, llava_image] + btn_list
+            ).then(LLAVA.http_bot, [llava_state, model_selector, temperature, top_p, max_output_tokens],
+                [llava_state, llava_chatbot] + btn_list)
+        llava_submit_btn.click(LLAVA.add_text, [llava_state, llava_textbox, llava_image, image_process_mode], [llava_state, llava_chatbot, llava_textbox, llava_image] + btn_list
+            ).then(LLAVA.http_bot, [llava_state, model_selector, temperature, top_p, max_output_tokens],
+                [llava_state, llava_chatbot] + btn_list)
+
+        if args.model_list_mode == "once":
+            raise ValueError(f"Unsupported model list mode: {args.model_list_mode}")
+        elif args.model_list_mode == "reload":
+            print('disable for debugging')
+            demo.load(LLAVA.load_demo_refresh_model_list, inputs=None, 
+                      outputs=[llava_state, model_selector]
+                      ).then(switch_to_compose, [],  [task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options] #first tab show doesn't need any
+                      ).then(GLIGEN.clear, inputs=[task, sketch_pad_trigger, batch_size, gligen_state],
+                                outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, gligen_state], queue=False)
+
+        else:
+            raise ValueError(f"Unknown model list mode: {args.model_list_mode}")        
+
+        gligen.select(
+                switch_to_generate, 
+                    inputs=[], 
+                    outputs=[task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options])
+        gligen_inpaint.select(
+                switch_to_inpaint, 
+                    inputs=[], 
+                    outputs=[task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options],
+                    queue=False)
+
+        compose_tab.select(
+                switch_to_compose, [],  [task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options])
+
+    return demo
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--port", type=int)
+    parser.add_argument("--controller-url", type=str, default="http://localhost:10000")
+    parser.add_argument("--concurrency-count", type=int, default=8)
+    parser.add_argument("--model-list-mode", type=str, default="reload",
+        choices=["once", "reload"])
+    parser.add_argument("--share", action="store_true")
+    parser.add_argument("--moderate", action="store_true")
+    parser.add_argument("--embed", action="store_true")
+    args = parser.parse_args()
+    LLAVA.set_args(args)
+
+    demo = build_demo()
+    demo.queue(concurrency_count=1, api_open=True)
+    demo.launch()

lama_predict.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+# Example command:
+# ./bin/predict.py \
+#       model.path=<path to checkpoint, prepared by make_checkpoint.py> \
+#       indir=<path to input data> \
+#       outdir=<where to store predicts>
+
+import logging
+import os
+import sys
+import traceback
+
+from saicinpainting.evaluation.utils import move_to_device
+from saicinpainting.evaluation.refinement import refine_predict
+os.environ['OMP_NUM_THREADS'] = '1'
+os.environ['OPENBLAS_NUM_THREADS'] = '1'
+os.environ['MKL_NUM_THREADS'] = '1'
+os.environ['VECLIB_MAXIMUM_THREADS'] = '1'
+os.environ['NUMEXPR_NUM_THREADS'] = '1'
+
+import cv2
+import hydra
+import numpy as np
+import torch
+import tqdm
+import yaml
+from omegaconf import OmegaConf
+from torch.utils.data._utils.collate import default_collate
+
+from saicinpainting.training.data.datasets import make_default_val_dataset
+from saicinpainting.training.trainers import load_checkpoint
+from saicinpainting.utils import register_debug_signal_handlers
+
+LOGGER = logging.getLogger(__name__)
+
+
+#@hydra.main(config_path='../configs/prediction', config_name='web_server.yaml')
+def main(predict_config: dict):
+    try:
+        #register_debug_signal_handlers()  # kill -10 <pid> will result in traceback dumped into log
+
+        device = torch.device(predict_config.device)
+
+        train_config_path = os.path.join(predict_config.model.path, 'config.yaml')
+        with open(train_config_path, 'r') as f:
+            train_config = OmegaConf.create(yaml.safe_load(f))
+        
+        train_config.training_model.predict_only = True
+        train_config.visualizer.kind = 'noop'
+
+        out_ext = predict_config.get('out_ext', '.png')
+
+        checkpoint_path = os.path.join(predict_config.model.path, 
+                                       'models', 
+                                       predict_config.model.checkpoint)
+        model = load_checkpoint(train_config, checkpoint_path, strict=False, map_location='cpu')
+        model.freeze()
+        if not predict_config.get('refine', False):
+            model.to(device)
+
+        if not predict_config.indir.endswith('/'):
+            predict_config.indir += '/'
+
+        dataset = make_default_val_dataset(predict_config.indir, **predict_config.dataset)
+        for img_i in tqdm.trange(len(dataset)):
+            mask_fname = dataset.mask_filenames[img_i]
+            cur_out_fname = os.path.join(
+                predict_config.outdir, 
+                os.path.splitext(mask_fname[len(predict_config.indir):])[0] + out_ext
+            )
+            os.makedirs(os.path.dirname(cur_out_fname), exist_ok=True)
+            batch = default_collate([dataset[img_i]])
+            if predict_config.get('refine', False):
+                assert 'unpad_to_size' in batch, "Unpadded size is required for the refinement"
+                # image unpadding is taken care of in the refiner, so that output image
+                # is same size as the input image
+                cur_res = refine_predict(batch, model, **predict_config.refiner)
+                cur_res = cur_res[0].permute(1,2,0).detach().cpu().numpy()
+            else:
+                with torch.no_grad():
+                    batch = move_to_device(batch, device)
+                    batch['mask'] = (batch['mask'] > 0) * 1
+                    batch = model(batch)                    
+                    cur_res = batch[predict_config.out_key][0].permute(1, 2, 0).detach().cpu().numpy()
+                    unpad_to_size = batch.get('unpad_to_size', None)
+                    if unpad_to_size is not None:
+                        orig_height, orig_width = unpad_to_size
+                        cur_res = cur_res[:orig_height, :orig_width]
+
+            cur_res = np.clip(cur_res * 255, 0, 255).astype('uint8')
+            cur_res = cv2.cvtColor(cur_res, cv2.COLOR_RGB2BGR)
+            cv2.imwrite(cur_out_fname, cur_res)
+
+    except KeyboardInterrupt:
+        LOGGER.warning('Interrupted by user')
+    except Exception as ex:
+        LOGGER.critical(f'Prediction failed due to {ex}:\n{traceback.format_exc()}')
+        sys.exit(1)
+
+
+#if __name__ == '__main__':
+#    main()

lama_server.py

@@ -0,0 +1,84 @@
+
+from flask import Flask, jsonify, send_file, request
+import base64
+from PIL import Image, ImageOps
+import io
+
+import hydra
+from omegaconf import DictConfig
+from lama_predict import main as lama_predict
+
+import os
+import yaml
+from omegaconf import OmegaConf
+
+cwd = os.getcwd()
+print(cwd)
+
+config_path = os.path.join(cwd, "configs/prediction/default.yaml")
+with open(config_path, 'r') as f:
+  config = OmegaConf.create(yaml.safe_load(f))
+
+config.model.path = os.path.join(cwd, "big-lama")
+config.indir = os.path.join(cwd, "web_server_input")
+config.outdir = os.path.join(cwd, "web_server_output")
+config.refine = False
+
+app = Flask(__name__)
+
+@app.route("/api/v2/image", methods=["GET", "POST"])
+def echo_image():
+  # Get the image data from the request body
+  json_dict = request.get_json()
+  print(type(json_dict))
+  # Get the value of the "image" key, which is the base64 encoded image data
+  base64_image_data = json_dict["image"]
+  #print(base64_image_data[0:500])
+
+  image_bytes = base64.b64decode(base64_image_data)
+  image_stream = io.BytesIO(image_bytes)
+  image = Image.open(image_stream)
+  print(image.format_description)
+  if not os.path.exists("web_server_input"):
+    os.makedirs("web_server_input")
+  image.save("web_server_input/server.png")
+
+  base64_mask_data = json_dict["mask"]  
+  image_bytes = base64.b64decode(base64_mask_data)
+  image_stream = io.BytesIO(image_bytes)
+  mask = Image.open(image_stream)
+  print(mask.format_description)
+  print(mask.format)
+  print(mask.size)
+  print(mask.mode)
+  if (mask.mode != "L"):
+    mask = mask.convert("L")
+  if not os.path.exists("web_server_input"):
+    os.makedirs("web_server_input")
+  mask.save("web_server_input/server_mask.png")
+
+  # Apply the mask to the image
+  # Create a new transparent image with the same size and mode as the image
+  transparent = Image.new(image.mode, image.size, (0, 0, 0, 0))
+  # Composite the image and the transparent image using the mask
+  masked_image = Image.composite(image, transparent, mask)
+  masked_image.save("server_masked_image.png")  
+
+  # Convert the masked image to bytes and create a new stream
+  masked_image_stream = io.BytesIO()
+  masked_image.save(masked_image_stream, format='PNG')
+  masked_image_stream.seek(0)
+
+  lama_predict(config)
+
+  with open("web_server_output/server_mask.png", "rb") as image_file:
+    image_bytes = image_file.read() 
+    image_inpainted_stream = io.BytesIO(image_bytes)
+    print(image.format_description)  
+  image_inpainted_stream.seek(0)
+
+  return send_file(image_inpainted_stream, mimetype="image/png")
+
+if __name__ == "__main__":
+  app.run(debug=True, port=9171)
+

llava_interactive.py

@@ -0,0 +1,705 @@
+import argparse
+import base64
+import io
+import os
+import sys
+
+import cv2
+import gradio as gr
+import numpy as np
+import requests
+from functools import partial                
+from PIL import Image, ImageOps
+
+sys.path.append(os.path.join(os.environ['LLAVA_INTERACTIVE_HOME'], 'GLIGEN/demo'))
+import GLIGEN.demo.app as GLIGEN
+sys.path.append(os.path.join(os.environ['LLAVA_INTERACTIVE_HOME'], 'SEEM/demo_code'))
+import SEEM.demo_code.app as SEEM #must import GLIGEN_app before this. Otherwise, it will hit a protobuf error
+sys.path.append(os.path.join(os.environ['LLAVA_INTERACTIVE_HOME'], 'LLaVA'))
+import LLaVA.llava.serve.gradio_web_server as LLAVA
+
+class ImageMask(gr.components.Image):
+    """
+    Sets: source="canvas", tool="sketch"
+    """
+
+    is_template = True
+
+    def __init__(self, **kwargs):
+        super().__init__(source="upload", tool="sketch", interactive=True, **kwargs)
+
+    def preprocess(self, x):
+        if isinstance(x, str):
+            x = {'image': x, 'mask': x}
+        elif isinstance(x, dict):            
+            if (x['mask'] is None and x['image'] is None):
+                x
+            elif (x['image'] is None):
+                x['image'] = str(x['mask'])
+            elif (x['mask'] is None):
+                x['mask'] = str(x['image']) #not sure why mask/mask is None sometimes, this prevents preprocess crashing
+        elif x is not None:
+            assert False, 'Unexpected type {0} in ImageMask preprocess()'.format(type(x))
+
+        return super().preprocess(x)
+
+css = """
+#compose_btn {
+    --tw-border-opacity: 1;
+    border-color: rgb(255 216 180 / var(--tw-border-opacity));
+    --tw-gradient-from: rgb(255 216 180 / .7);
+    --tw-gradient-to: rgb(255 216 180 / 0);
+    --tw-gradient-stops: var(--tw-gradient-from), var(--tw-gradient-to);
+    --tw-gradient-to: rgb(255 176 102 / .8);
+    --tw-text-opacity: 1;
+    color: rgb(238 116 0 / var(--tw-text-opacity));
+}
+"""
+
+def get_bounding_box(img):
+    # Get the indices of all non-zero pixels
+    if (np.any(img) == False): #protect agaist an empty img
+        return None
+    non_zero_indices = np.nonzero(img)
+
+    # Get the minimum and maximum indices for each axis
+    min_x = np.min(non_zero_indices[1])
+    max_x = np.max(non_zero_indices[1])
+    min_y = np.min(non_zero_indices[0])
+    max_y = np.max(non_zero_indices[0])
+
+    # Return the bounding box as a tuple of (min_x, min_y, max_x, max_y)
+    return (min_x, min_y, max_x, max_y)
+
+def composite_all_layers(base, objects): #debugging use only
+    img = base.copy()
+    for obj in objects:
+        for i in range(obj['img'].shape[0]):
+            for j in range(obj['img'].shape[1]):
+                if obj['img'][i, j, 3] != 0:
+                    img[i, j] = obj['img'][i, j]
+    return img
+
+def changed_objects_handler(mask_dilate_slider, state, evt: gr.SelectData):
+    state['move_no'] += 1
+
+    pos_x, pos_y = evt.index #obj moved out of scene is signaled by (10000, 10000)
+    obj_id = 255 - evt.value
+    print(f"obj {obj_id} moved by {pos_x}, {pos_y}")
+
+    img = state['base_layer']
+    for obj in state['changed_objects']:
+        if obj['id'] == obj_id:
+            img = obj['img']
+            state['changed_objects'].remove(obj)
+            break
+
+    new_img = np.zeros_like(img)
+    bbox = None
+    for i in range(img.shape[0]):
+        for j in range(img.shape[1]):
+            if img[i, j, 3] == obj_id:
+                new_i = i + pos_y
+                new_j = j + pos_x
+                if new_i >= 0 and new_i < img.shape[0] and new_j >= 0 and new_j < img.shape[1]:
+                    new_img[new_i, new_j] = img[i, j]                        
+                img[i, j] = 0
+
+    bbox = get_bounding_box(new_img) #returns None if obj moved out of scene
+    print("bbox: ", bbox)
+    state['changed_objects'].append({'id': obj_id, 'img': new_img, 'text': state['segment_info'][obj_id], 'box': bbox})
+
+    #Enable for debugging only. See if the composited image is correct.
+    #composed_img_updated = composite_all_layers(state['base_layer'], state['changed_objects'])    
+    #filename = str(f"composited_imge_{state['move_no']}") + ".png"
+    #cv2.imwrite(filename, composed_img_updated[:, :, 0:3])
+
+
+    return mask_dilate_slider, state['base_layer_masked'], state
+
+def get_base_layer_mask(state):
+
+    changed_obj_id = []
+    for obj in state['changed_objects']:
+        changed_obj_id.append(obj['id'])
+
+    #union of mask of all objects
+    img = state['orignal_segmented']
+    mask = np.zeros(img.shape[:2], dtype=np.uint8)
+    for i in range(img.shape[0]):
+        for j in range(img.shape[1]):
+            if img[i, j, 3] in changed_obj_id:
+                mask[i, j] = 255
+    state['base_layer_mask'] = mask
+
+    mask_image = Image.fromarray(mask)
+    if (mask_image.mode != "L"):
+        mask_image = mask_image.convert("L")
+    mask_image = ImageOps.invert(mask_image)      
+    #mask_image.save("mask_image.png")      
+
+    img = state['orignal_segmented']
+    orig_image = Image.fromarray(img[:,:,:3])
+    orig_image.save("orig_image.png")          
+    transparent = Image.new(orig_image.mode, orig_image.size, (0, 0, 0, 0))
+    masked_image = Image.composite(orig_image, transparent, mask_image)
+    #masked_image.save("get_masked_background_image.png")  
+
+    return masked_image, state
+
+def get_inpainted_background(state, mask_dilate_slider):
+
+    # Define the URL of the REST API endpoint
+    url = "http://localhost:9171/api/v2/image"
+
+    img = state['orignal_segmented']
+    if (isinstance(img, Image.Image) is not True):
+        img = Image.fromarray(img)
+    # Create a BytesIO object and save the image there
+    buffer = io.BytesIO()
+    img.save(buffer, format="PNG")
+    # Get the bytes value from the buffer
+    img_bytes = buffer.getvalue()
+
+    encoded_string = base64.b64encode(img_bytes).decode("utf-8")
+
+    if (mask_dilate_slider != 0) :
+        mask = state['base_layer_mask_enlarged']
+    else:
+        mask = state['base_layer_mask']
+    if (isinstance(mask, Image.Image) is not True):
+        mask = Image.fromarray(mask)
+    
+    #mask has background as 1, lama needs object to be 1
+    if (mask.mode != "L"):
+        mask = mask.convert("L")
+    mask = ImageOps.invert(mask)     
+
+    # Create a BytesIO object and save the image there
+    buffer = io.BytesIO()
+    mask.save(buffer, format="PNG")
+    # Get the bytes value from the buffer
+    mask_bytes = buffer.getvalue()
+
+    encoded_string_mask = base64.b64encode(mask_bytes).decode("utf-8")
+
+
+    # Create a POST request to the endpoint
+    headers = {"Content-Type": "application/json"}
+    data = {"image": encoded_string, "mask": encoded_string_mask}
+    response = requests.post(url, headers=headers, json=data)
+
+    # Check the status code of the response
+    if response.status_code == 200:
+        # The request was successful
+        print("Image received successfully")
+        image_data = response.content
+        # Create a io.BytesIO object from the image data
+        dataBytesIO = io.BytesIO(image_data)
+        # Open the image using Image.open()
+        image = Image.open(dataBytesIO)
+        #image.save("lama_returned_image.png")
+
+    else:
+        # The request failed
+        print("Error: HTTP status code {}".format(response.status_code))
+        print(response.text)    
+
+    return image
+
+def get_enlarged_masked_background(state, mask_dilate_slider):
+
+    mask = state['base_layer_mask']
+
+    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (mask_dilate_slider, mask_dilate_slider))
+    mask_dilated = cv2.dilate(mask, kernel)
+
+    #mask the original
+    mask_image = Image.fromarray(mask_dilated)
+    if (mask_image.mode != "L"):
+        mask_image = mask_image.convert("L")
+    mask_image = ImageOps.invert(mask_image)     
+    state['base_layer_mask_enlarged'] = mask_image     
+    #mask_image.save("enlarged_mask_image.png")      
+
+    img = state['orignal_segmented']
+    orig_image = Image.fromarray(img[:,:,:3])
+    transparent = Image.new(orig_image.mode, orig_image.size, (0, 0, 0, 0))
+    masked_image = Image.composite(orig_image, transparent, mask_image)
+    #masked_image.save("enlarged_masked_background_image.png")  
+
+    return masked_image, state
+
+def get_base_layer_inpainted(state, mask_dilate_slider): 
+    masked_img, state = get_enlarged_masked_background(state, mask_dilate_slider)
+    inpainted_img = get_inpainted_background(state, mask_dilate_slider)
+    state['base_layer_inpainted'] = np.array(inpainted_img)
+    return masked_img, inpainted_img, state
+
+def log_image_and_mask(img, mask): #for debugging use only
+    counter = 0
+    for filename in os.listdir('.'):
+        if filename.startswith('img_') and filename.endswith('.png'):
+            try:
+                num = int(filename[4:-4])
+                if num > counter:
+                    counter = num
+            except ValueError:
+                pass
+    counter += 1
+    cv2.imwrite(f"img_{counter}.png", img)
+    cv2.imwrite(f"img_{counter}_mask.png", mask.astype(np.uint8) * 255)
+
+def get_segments (img, task, reftxt, mask_dilate_slider, state):
+    assert (isinstance(state, dict))    
+    state['orignal_segmented'] = None
+    state['base_layer'] = None
+    state['base_layer_masked'] = None
+    state['base_layer_mask'] = None
+    state['base_layer_mask_enlarged'] = None
+    state['base_layer_inpainted'] = None
+    state['segment_info'] = None
+    state['seg_boxes'] = {}
+    state['changed_objects'] = []
+    state['move_no'] = 0
+
+    print("Calling SEEM_app.inference")    
+
+    if isinstance(img['image'], np.ndarray):
+        pil_image = Image.fromarray(img['image'])
+    if isinstance(img['mask'], np.ndarray):
+        pil_mask = Image.fromarray(img['mask'])
+    img = {'image': pil_image, 'mask': pil_mask}
+    img_ret, seg_info = SEEM.inference (img, task, reftxt=reftxt)
+    #SEEM doesn't always respect the input img dimentions
+    tgt_size=(img['image'].width, img['image'].height)
+    img_ret = img_ret.resize(tgt_size, resample=Image.Resampling.NEAREST)
+    state['orignal_segmented'] = np.array(img_ret).copy()
+    state['base_layer'] = np.array(img_ret)
+    state['segment_info'] = seg_info
+    img_ret_array = np.array(img_ret)
+    img_ret_array[:,:,3] = 255 - img_ret_array[:,:,3]
+    #NOTE: if write out as a png, the pixels values get messed up. Same reason the client side colors look weird.
+    #cv2.imwrite(f"get_segments_img_ret.bmp", img_ret_array)
+
+
+    for obj_id, lable in seg_info.items():
+        obj_img = (img_ret_array[:,:,3] == 255 - obj_id)
+        #cv2.imwrite(f"img_{obj_id}.png", obj_img.astype(np.uint8) * 255)
+        #log_image_and_mask(np.array(img['image']), obj_img)
+        bbox = get_bounding_box(obj_img)
+        print(f"obj_id={obj_id}, lable={lable}, bbox={bbox}")
+        state['seg_boxes'][obj_id] = bbox
+
+    #add a special event, obj stays at the original spot 
+    data = {}
+    data["index"] = (0, 0)
+    data["value"] = 254 # ==> 1, the only object allowed for now
+    data["selected"] = True
+    evt = gr.SelectData(None, data)
+    mask_dilate_slider, _, state = changed_objects_handler(mask_dilate_slider, state, evt)
+
+    state['base_layer_masked'], state = get_base_layer_mask(state)
+    if (mask_dilate_slider != 0):
+        enlarged_masked_background, state = get_enlarged_masked_background(state, mask_dilate_slider)
+    state['base_layer_inpainted'] = np.array(get_inpainted_background(state, mask_dilate_slider))
+
+    return Image.fromarray(img_ret_array), enlarged_masked_background, state['base_layer_inpainted'], state
+
+def get_generated(grounding_text, fix_seed, rand_seed, state):
+
+    if ('base_layer_inpainted' in state) == False :
+        raise gr.Error('The segmentation step must be completed first before generating a new image')
+
+    inpainted_background_img = state['base_layer_inpainted']
+    assert inpainted_background_img is not None, 'base layer should be inpainted after segment'
+
+    state['boxes'] = []
+    for items in state['changed_objects']:
+        if items['box'] is not None:
+            state['boxes'].append(items['box'])
+
+    if (len(state['boxes']) == 0):
+        if (len(grounding_text) != 0):
+            grounding_text = []            
+            print("No grounding box found. Grounding text will be ignored.")
+        return inpainted_background_img.copy(), state, None
+    
+    print('Calling GLIGEN_app.generate')
+    print('grounding_text: ', grounding_text)
+    print(state['boxes'], len(state['boxes']))
+    assert len(state['boxes']) == 1, 'Only handle one segmented object at a time'
+    if (len(grounding_text) == 0): #mostly user forgot to drag the object and didn't provide grounding text
+         raise gr.Error('Please providing grounding text to match the identified object')
+    out_gen_1, _, _, _, state = GLIGEN.generate(task='Grounded Inpainting', language_instruction='',
+                        grounding_texts=grounding_text, sketch_pad=inpainted_background_img,
+                        alpha_sample=0.3, guidance_scale=7.5, batch_size=1, 
+                        fix_seed=fix_seed, rand_seed=rand_seed, use_actual_mask=False, append_grounding=True,
+                        style_cond_image=None, inpainting_image=inpainted_background_img, inpainting_mask=None, state=state)
+
+    return out_gen_1['value'], state
+
+def get_generated_full(task, language_instruction, grounding_instruction, sketch_pad,
+                        alpha_sample, guidance_scale, batch_size,
+                        fix_seed, rand_seed,
+                        use_actual_mask,
+                        append_grounding, style_cond_image,
+                        state):
+    
+    out_gen_1, _, _, _, state = GLIGEN.generate(
+                            task, language_instruction, grounding_instruction, sketch_pad,
+                            alpha_sample, guidance_scale, batch_size,
+                            fix_seed, rand_seed,
+                            use_actual_mask,
+                            append_grounding, style_cond_image,
+                            state)
+    return out_gen_1['value'], state
+
+def gligen_change_task(state):
+    if (state['working_image'] is not None):
+        task = "Grounded Inpainting"
+    else:
+        task = "Grounded Generation"
+    return task
+
+def clear_sketch_pad_mask(sketch_pad_image):
+    sketch_pad = ImageMask.update(value=sketch_pad_image, visible=True) 
+    return sketch_pad
+
+def save_shared_state(img, state):
+    if (isinstance(img, dict) and 'image' in img):
+        state['working_image'] = img['image']
+    else:
+        state['working_image'] = img
+    return state
+
+def load_shared_state(state, task = None):
+    if (task == "Grounded Generation"):
+        return None
+    else:
+        return state['working_image']
+
+def update_shared_state(state, task):
+    if (task == "Grounded Generation"):
+        state['working_image'] = None
+    return state
+
+def update_sketch_pad_trigger(sketch_pad_trigger, task):
+    if (task == "Grounded Generation"):
+        sketch_pad_trigger = sketch_pad_trigger + 1
+    return sketch_pad_trigger
+
+def clear_grounding_info(state):
+    state['boxes'] = []
+    state['masks'] = []
+    return state, ''
+
+def switch_to_generate ():
+    task = "Grounded Generation"
+    return task, gr.Image.update(visible=True), gr.Textbox.update(visible=True), gr.Textbox.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Accordion.update(visible=True)
+
+def switch_to_inpaint ():
+    task = "Grounded Inpainting"
+    return task, gr.Image.update(visible=True), gr.Textbox.update(visible=False), gr.Textbox.update(visible=True), gr.Button.update(visible=True), gr.Button.update(visible=True), gr.Accordion.update(visible=True)
+
+def switch_to_compose ():
+    task = "Compose"
+    return task, gr.Image.update(visible=False), gr.Textbox.update(visible=False), gr.Textbox.update(visible=False), gr.Button.update(visible=False), gr.Button.update(visible=False), gr.Accordion.update(visible=False)
+
+def copy_to_llava_input(img):
+    print('WORKING IMAGE CHANGED!!!!')
+    if (isinstance(img, Image.Image) is not True):
+        img = Image.fromarray(img)
+    return img
+                
+def build_demo():
+    demo = gr.Blocks(title="🌋 LLaVA-Interactive", css=css+GLIGEN.css)
+    with demo:
+        compose_state = gr.State({'boxes': [], 'move_no': 0, 'base_layer': None, 'segment_info': None, 'seg_boxes': {}, 'changed_objects': []})
+        llava_state = gr.State()
+        shared_state = gr.State({'working_image': None})
+        gligen_state = gr.State({'draw_box': True})
+
+        gr.Markdown('<h1 style="text-align: center;"></h1>')
+        gr.Markdown('<h1 style="text-align: center;">LLaVA Interactive</h1>')
+        gr.Markdown('<h1 style="text-align: center;"></h1>')
+
+        gr.Markdown('**Experience interactive multimodal chatting and image manipulation. Select a tab for your task and follow the instructions. Switch tasks anytime and ask questions in the chat window.**')
+
+        with gr.Row(visible=False):
+            working_image = gr.Image(label="Working Image", type="numpy", elem_id="working_image", visible=False, interactive=False) #hidden image to save current working image
+            #for gligen
+            sketch_pad_trigger = gr.Number(value=0, visible=False)
+            sketch_pad_resize_trigger = gr.Number(value=0, visible=False)
+            init_white_trigger = gr.Number(value=0, visible=False)
+            image_scale = gr.Number(value=0, elem_id="image_scale", visible=False)
+            task = gr.Radio(
+                choices=["Grounded Generation", 'Grounded Inpainting', 'Compose'],
+                type="value",
+                value="Grounded Inpainting",
+                label="Task",
+                visible=False
+            )
+            
+        with gr.Row(equal_height=False):
+            with gr.Column():
+
+                with gr.Row():
+                    sketch_pad = ImageMask(label="Sketch Pad", type="numpy", shape=(512, 512), width=384, elem_id="img2img_image", brush_radius=20.0, visible=True)
+
+                compose_tab = gr.Tab("Remove or Change Objects")
+                with compose_tab:
+                    gr.Markdown("Segment an object by drawing a stroke or giving a referring text. Then press the segment button. Drag the highlighted object to move it. To remove it, drag it out of the frame. To replace it with a new object, give an instruction only if the object is removed and press the generate button until you like the image.")
+                    with gr.Row().style(equal_height=False):
+                        with gr.Column():
+                            with gr.Group():
+                                with gr.Column():
+                                    with gr.Row():
+                                        segment_task= gr.Radio(["Stroke", "Text"], value="Stroke", label='Choose segmentation method')
+                                        segment_text = gr.Textbox(label="Enter referring text")
+                                    segment_btn = gr.Button("Segment", elem_id="segment-btn")
+
+                            with gr.Group():
+                                segmented_img = gr.Image(label="Move or delete object", tool="compose", height=256) 
+
+                            with gr.Group():
+                                with gr.Column():
+                                    grounding_text_box = gr.Textbox(label="Enter grounding text for generating a new image")
+                                    with gr.Row():
+                                        compose_clear_btn = gr.Button("Clear", elem_id="compose_clear_btn")
+                                        compose_btn = gr.Button("Generate", elem_id="compose_btn")
+
+                            with gr.Accordion("Advanced Options", open=False):
+                                with gr.Row():
+                                    masked_background_img = gr.Image(label="Background", type='pil', interactive=False, height=256)
+                                    inpainted_background_img = gr.Image(label="Inpainted Background", type='pil', interactive=False, height=256)
+                                mask_dilate_slider = gr.Slider(minimum=0.0, maximum=100, value=50, step=2, interactive=True, label="Mask dilation",visible=True, scale=20)
+                                with gr.Row(visible=False):
+                                    compose_fix_seed = gr.Checkbox(value=False, label="Fixed seed", visible=False)
+                                    compose_rand_seed = gr.Slider(minimum=0, maximum=1000, step=1, value=0, label="Seed", visible=False)
+
+                gligen_inpaint = gr.Tab("Inpaint New Objects")
+                with gligen_inpaint:
+                    gr.Markdown("Add a new object to the image by drawing its bounding box and giving an instruction. Press the “generate” button repeatedly until you like the image. Press “clear” to accept the image and start over with another object.")
+
+                gligen = gr.Tab("Generate New Image")
+                with gligen:
+                    gr.Markdown("Generate a new image by giving a language instruction below. Draw a bounding box and give an instruction for any specific objects that need to be grounded in certain places. Hit the “generate” button repeatedly until you get the image you want.")    
+
+                with gr.Group(visible=False):
+                    language_instruction = gr.Textbox(label="Language instruction", elem_id='language_instruction', visible=False)
+                    grounding_instruction = gr.Textbox(label="Grounding instruction (Separated by semicolon)", elem_id='grounding_instruction', visible=False)
+                    with gr.Row():
+                        gligen_clear_btn = gr.Button(value='Clear', visible=False)
+                        gligen_gen_btn = gr.Button(value='Generate', elem_id="generate-btn", visible=False)
+
+                with gr.Group():
+                    out_imagebox = gr.Image(type="pil", label="Parsed Sketch Pad", height=256, visible=False)
+
+                gligen_adv_options = gr.Accordion("Advanced Options", open=False, visible=False)
+                with gligen_adv_options:
+                    with gr.Column():
+                        alpha_sample = gr.Slider(minimum=0, maximum=1.0, step=0.1, value=0.3, label="Scheduled Sampling (τ)")
+                        guidance_scale = gr.Slider(minimum=0, maximum=50, step=0.5, value=7.5, label="Guidance Scale")
+
+                with gr.Row(visible=False):
+                    batch_size = gr.Slider(minimum=1, maximum=4, step=1, value=1, label="Number of Samples", visible=False)
+                    append_grounding = gr.Checkbox(value=True, label="Append grounding instructions to the caption",visible=False)
+                    use_actual_mask = gr.Checkbox(value=False, label="Use actual mask for inpainting", visible=False)
+                    fix_seed = gr.Checkbox(value=False, label="Fixed seed",visible=False)
+                    rand_seed = gr.Slider(minimum=0, maximum=1000, step=1, value=0, label="Seed",visible=False)
+                    use_style_cond = gr.Checkbox(value=False, label="Enable Style Condition",visible=False)
+                    style_cond_image = gr.Image(type="pil", label="Style Condition", visible=False, interactive=False)
+
+                controller = GLIGEN.Controller()
+                sketch_pad.edit(
+                    GLIGEN.draw,
+                    inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, gligen_state],
+                    outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, gligen_state],
+                    queue=False,
+                )
+                llava_image = gr.Image(label='sketch_pad_image', type='pil', visible=False, interactive=False)
+                working_image.change(copy_to_llava_input, [working_image], [llava_image])
+                sketch_pad.upload(
+                    save_shared_state, 
+                        inputs = [sketch_pad, shared_state], 
+                        outputs = shared_state).then(
+                    load_shared_state, [shared_state], working_image)
+                grounding_instruction.change(
+                    GLIGEN.draw,
+                    inputs=[task, sketch_pad, grounding_instruction, sketch_pad_resize_trigger, gligen_state],
+                    outputs=[out_imagebox, sketch_pad_resize_trigger, image_scale, gligen_state],
+                    queue=False,
+                )
+                gligen_clear_btn.click(
+                    GLIGEN.clear,
+                    inputs=[task, sketch_pad_trigger, batch_size, gligen_state],
+                    outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, gligen_state],
+                    queue=False).then(
+                    clear_grounding_info, gligen_state, [gligen_state, grounding_instruction]).then(
+                    load_shared_state, [shared_state], sketch_pad).then(
+                    update_sketch_pad_trigger, [sketch_pad_trigger, task], sketch_pad_trigger)
+                task.change(
+                    partial(GLIGEN.clear, switch_task=True),
+                    inputs=[task, sketch_pad_trigger, batch_size, gligen_state],
+                    outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, gligen_state],
+                    queue=False).then(
+                    load_shared_state, [shared_state, task], sketch_pad).then(
+                    update_sketch_pad_trigger, [sketch_pad_trigger, task], sketch_pad_trigger).then(
+                    clear_grounding_info, gligen_state, [gligen_state, grounding_instruction])
+                sketch_pad_trigger.change(
+                    controller.init_white,
+                    inputs=[init_white_trigger],
+                    outputs=[sketch_pad, image_scale, init_white_trigger],
+                    queue=False)
+                sketch_pad_resize_trigger.change(
+                    controller.resize_masked,
+                    inputs=[gligen_state],
+                    outputs=[sketch_pad, gligen_state],
+                    queue=False)
+                        
+                gligen_gen_btn.click(
+                    get_generated_full,
+                    inputs=[
+                        task, language_instruction, grounding_instruction, sketch_pad,
+                        alpha_sample, guidance_scale, batch_size,
+                        fix_seed, rand_seed,
+                        use_actual_mask,
+                        append_grounding, style_cond_image,
+                        gligen_state],
+                    outputs=[sketch_pad, gligen_state], 
+                    queue=True).then(
+                    save_shared_state, [sketch_pad, shared_state], shared_state).then(
+                    load_shared_state, [shared_state], working_image)
+                
+                sketch_pad_resize_trigger.change(
+                    None,
+                    None,
+                    sketch_pad_resize_trigger,
+                    _js=GLIGEN.rescale_js,
+                    queue=False)
+                init_white_trigger.change(
+                    None,
+                    None,
+                    init_white_trigger,
+                    _js=GLIGEN.rescale_js,
+                    queue=False)
+                use_style_cond.change(
+                    lambda cond: gr.Image.update(visible=cond),
+                    use_style_cond,
+                    style_cond_image,
+                    queue=False)
+                task.change(
+                    controller.switch_task_hide_cond,
+                    inputs=task,
+                    outputs=[use_style_cond, style_cond_image, alpha_sample, use_actual_mask],
+                    queue=False)
+
+
+            with gr.Column():
+                gr.Markdown("Chat with the latest image on the left at any time by entering your text below.")
+                llava_chatbot = gr.Chatbot(elem_id="chatbot", label="LLaVA Chatbot", height=750)
+                with gr.Column(scale=8):
+                    llava_textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False)
+                with gr.Column(scale=1, min_width=60):
+                    llava_submit_btn = gr.Button(value="Submit", visible=False)
+
+                with gr.Row(visible=False):
+                    upvote_btn = gr.Button(value="👍  Upvote", interactive=False, visible=False)
+                    downvote_btn = gr.Button(value="👎  Downvote", interactive=False, visible=False)
+                    flag_btn = gr.Button(value="⚠️  Flag", interactive=False, visible=False)
+                    regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False, visible=False)
+                    llava_clear_btn = gr.Button(value="🗑️  Clear history", interactive=False, visible=False)
+                    with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
+                        temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",visible=True)
+                        top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",visible=True)
+                        max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",visible=True)
+
+        segment_btn.click(get_segments, inputs=[sketch_pad, segment_task, segment_text, mask_dilate_slider, compose_state], outputs=[segmented_img, masked_background_img, inpainted_background_img, compose_state], queue=True)
+        segmented_img.select (changed_objects_handler, [mask_dilate_slider, compose_state], [mask_dilate_slider, masked_background_img, compose_state])
+        mask_dilate_slider.release(get_base_layer_inpainted, inputs=[compose_state, mask_dilate_slider], outputs=[masked_background_img, inpainted_background_img, compose_state])
+        compose_btn.click(get_generated, [grounding_text_box, compose_fix_seed, compose_rand_seed, compose_state], [sketch_pad, compose_state], queue=True).then(
+                          save_shared_state, [sketch_pad, shared_state], shared_state).then(
+                          load_shared_state, [shared_state], working_image)
+        compose_clear_btn.click(load_shared_state, [shared_state], sketch_pad)
+            
+        image_process_mode = gr.Radio(
+                    ["Crop", "Resize", "Pad"],
+                    value="Crop",
+                    label="Preprocess for non-square image",
+                    visible=False)
+        models = LLAVA.get_model_list(args)
+        model_selector = gr.Dropdown(
+            choices=models,
+            value=models[0] if len(models) > 0 else "",
+            interactive=True,
+            show_label=False,
+            container=False,
+            visible=False)
+
+        btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, llava_clear_btn]
+        upvote_btn.click(LLAVA.upvote_last_response,
+            [llava_state, model_selector], [llava_textbox, upvote_btn, downvote_btn, flag_btn])
+        downvote_btn.click(LLAVA.downvote_last_response,
+            [llava_state, model_selector], [llava_textbox, upvote_btn, downvote_btn, flag_btn])
+        flag_btn.click(LLAVA.flag_last_response,
+            [llava_state, model_selector], [llava_textbox, upvote_btn, downvote_btn, flag_btn])
+        regenerate_btn.click(LLAVA.regenerate, [llava_state, image_process_mode],
+            [llava_state, llava_chatbot, llava_textbox, sketch_pad] + btn_list).then(
+            LLAVA.http_bot, [llava_state, model_selector, temperature, top_p, max_output_tokens],
+            [llava_state, llava_chatbot] + btn_list)
+        llava_clear_btn.click(LLAVA.clear_history, None, [llava_state, llava_chatbot, llava_textbox, llava_image] + btn_list)
+        
+        llava_textbox.submit(LLAVA.add_text, [llava_state, llava_textbox, llava_image, image_process_mode], [llava_state, llava_chatbot, llava_textbox, llava_image] + btn_list
+            ).then(LLAVA.http_bot, [llava_state, model_selector, temperature, top_p, max_output_tokens],
+                [llava_state, llava_chatbot] + btn_list)
+        llava_submit_btn.click(LLAVA.add_text, [llava_state, llava_textbox, llava_image, image_process_mode], [llava_state, llava_chatbot, llava_textbox, llava_image] + btn_list
+            ).then(LLAVA.http_bot, [llava_state, model_selector, temperature, top_p, max_output_tokens],
+                [llava_state, llava_chatbot] + btn_list)
+
+        if args.model_list_mode == "once":
+            raise ValueError(f"Unsupported model list mode: {args.model_list_mode}")
+        elif args.model_list_mode == "reload":
+            print('disable for debugging')
+            demo.load(LLAVA.load_demo_refresh_model_list, inputs=None, 
+                      outputs=[llava_state, model_selector]
+                      ).then(switch_to_compose, [],  [task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options] #first tab show doesn't need any
+                      ).then(GLIGEN.clear, inputs=[task, sketch_pad_trigger, batch_size, gligen_state],
+                                outputs=[sketch_pad, sketch_pad_trigger, out_imagebox, image_scale, gligen_state], queue=False)
+
+        else:
+            raise ValueError(f"Unknown model list mode: {args.model_list_mode}")        
+
+        gligen.select(
+                switch_to_generate, 
+                    inputs=[], 
+                    outputs=[task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options])
+        gligen_inpaint.select(
+                switch_to_inpaint, 
+                    inputs=[], 
+                    outputs=[task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options],
+                    queue=False)
+
+        compose_tab.select(
+                switch_to_compose, [],  [task, out_imagebox, language_instruction, grounding_instruction, gligen_clear_btn, gligen_gen_btn, gligen_adv_options])
+
+    return demo
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", type=str, default="0.0.0.0")
+    parser.add_argument("--port", type=int)
+    parser.add_argument("--controller-url", type=str, default="http://localhost:10000")
+    parser.add_argument("--concurrency-count", type=int, default=8)
+    parser.add_argument("--model-list-mode", type=str, default="reload",
+        choices=["once", "reload"])
+    parser.add_argument("--share", action="store_true")
+    parser.add_argument("--moderate", action="store_true")
+    parser.add_argument("--embed", action="store_true")
+    args = parser.parse_args()
+    LLAVA.set_args(args)
+
+    demo = build_demo()
+    demo.queue(concurrency_count=1, api_open=False)
+    demo.launch()

requirements.txt

@@ -0,0 +1,51 @@
+albumentations==1.3.0
+accelerate==0.20.3
+altair==5.0.1
+cityscapesscripts==2.2.2
+diffusers==0.11.1
+diffdist==0.1
+ftfy==6.1.1
+fvcore==0.1.5.post20221221
+imageio==2.9.0
+imageio-ffmpeg==0.4.2
+invisible-watermark==0.1.5
+json_tricks==3.17.1
+kornia==0.6.9
+mup==1.0.0
+nltk==3.8.1
+numpy==1.23.1
+numba==0.57.1
+openai==0.27.8
+omegaconf==2.1.1
+opencv-python==4.7.0.72
+opencv-python-headless==4.7.0.72
+pandas==2.0.3
+pip==22.2.2
+pillow==9.4.0 
+pyarrow==12.0.1
+pycocotools==2.0.5 
+pydantic==1.10.9 
+pyyaml==6.0
+protobuf==3.20.3
+pytorch-lightning==1.4.2
+regex==2023.6.3
+scikit-image==0.20.0
+scikit-learn==1.2.2
+sentencepiece==0.1.99
+shapely==2.0.1
+scann==1.2.7
+streamlit==1.12.1
+timm==0.4.12
+--find-links https://download.pytorch.org/whl/cu117/torch_stable.html
+torch==2.0.1+cu117 
+--find-links https://download.pytorch.org/whl/cu117/torch_stable.html
+torchvision==0.15.2+cu117 
+test-tube==0.7.5
+transformers==4.28.0
+vision-datasets==0.2.2
+yacs==0.1.8
+clip @ git+https://github.com/openai/CLIP.git@a9b1bf5920416aaeaec965c25dd9e8f98c864f16
+openai-whisper @ git+https://github.com/openai/whisper.git@248b6cb124225dd263bb9bd32d060b6517e067f8
+einops @ git+https://github.com/arogozhnikov/einops.git
+detectron2 @ git+https://github.com/maureenzou/detectron2-xyz.git@42121d75e10d9f858f3a91b6a39f5722c02868f0
+gradio @ git+https://github.com/wchen-github/gradio.git

run_demo.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+pkill -9 -f llava.serve.controller
+pkill -9 -f llava.serve.model_worker
+pkill -9 -f lama_server
+pkill -9 -f llava_interactive
+
+eval "$(conda shell.bash hook)"
+
+(
+conda deactivate; \
+cd LLaVA; \
+pwd; \
+conda activate llava; \
+python -m llava.serve.controller --host 0.0.0.0 --port 10000 & \
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path ./llava-v1.5-13b &
+)
+
+sleep 30
+
+(
+conda deactivate; \
+conda activate lama; \
+cd lama; \
+pwd; \
+export TORCH_HOME=$(pwd) && export PYTHONPATH=$(pwd); \
+python ../lama_server.py & 
+) 
+
+sleep 10
+
+(
+conda deactivate; \
+conda activate llava_int; \
+export LLAVA_INTERACTIVE_HOME=.; \
+python llava_interactive.py
+)
+

setup.sh

@@ -0,0 +1,35 @@
+echo "Cloning dependent repos..." 
+git clone --single-branch https://github.com/wchen-github/GLIGEN.git
+git clone --single-branch https://github.com/wchen-github/Segment-Everything-Everywhere-All-At-Once.git SEEM
+git clone --single-branch https://github.com/wchen-github/LLaVA
+git clone --single-branch https://github.com/advimman/lama.git	
+
+
+echo "Creating environments and download pretrained models..."
+
+cd LLaVA
+conda create -n llava python=3.10 -y
+conda activate llava
+pip install --upgrade pip  # enable PEP 660 support
+pip install -e .
+#download pretrained model
+git clone https://huggingface.co/liuhaotian/llava-v1.5-13b
+conda deactivate
+cd ..
+
+#setting up lama
+cd lama
+conda env create --name lama -f conda_env.yml -y
+conda activate lama
+conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch -y
+pip install torch==1.10.2+cu113 --find-links https://download.pytorch.org/whl/cu113/torch_stable.html
+pip install torchvision==0.11.3+cu113 --find-links https://download.pytorch.org/whl/cu113/torch_stable.html
+pip install flask
+pip install pytorch-lightning
+#download pretrained model
+git clone https://huggingface.co/smartywu/big-lama download
+unzip download/big-lama.zip
+
+conda deactivate
+cd ..
+echo "Done setting up."