Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/demo.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/langloc.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

.gitignore

@@ -0,0 +1,166 @@
+*.pt
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Gradio
+.gradio

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,13 +1,58 @@
 ---
-title: Lang Ground
-emoji: 🔥
-colorFrom: pink
-colorTo: blue
+title: lang-ground
+app_file: app_langloc.py
 sdk: gradio
-sdk_version: 5.9.0
-app_file: app.py
-pinned: false
-license: apache-2.0
+sdk_version: 3.41.2
 ---
+# Language Grounding
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Localize and keep tracking things based on natural
+language sepecfication is a good idea but remains challenging due to the scarcity of large-scale annotated datasets.
+
+This repository provides a practical solution to identify relevant objects in the view and continuously tracking them, which is particularly beneficial for robotics applications.
+
+We offer several demos that showcase the following functionalities:
+
+## Language Localize
+Identify and locate objects based on natural language queries.
+
+Try our demo to experience an interesting use case: When analyzing food items, the model demonstrates contextual understanding:
+- For food nearing expiration: Suggests storing in the cabinet
+- For expired food: Recommends disposal in the trash can
+
+![langloc](/assets/langloc.jpg)
+
+
+
+
+## **Lang Ground (Localize + Track):** 
+
+Not only find but also continuously track objects of interest
+
+We prioritize making this project highly accessible and customizable:
+
+- **Open Box Design:** All components are modular and well-documented for easy understanding
+- **Customizable Pipeline:** Easily adapt the system for different use cases
+- **Extensible Framework:** Simple integration with other vision or language models
+
+
+## 🛠️ Install
+
+```bash
+git clone https://github.com/jing-bi/lang-ground.git && cd lang-ground
+
+mamba create -n lang-ground python=3.11
+mamba activate lang-ground
+
+pip install -e .
+```
+## Acknowledgments
+
+This project is built upon and inspired by the following repositories:
+
+- [Segment-Anything](https://github.com/facebookresearch/segment-anything-2)
+- [Supervision](https://github.com/roboflow/supervision)
+
+## License
+
+This project is licensed under the Apache 2.0 License

app_langloc.py

@@ -0,0 +1,88 @@
+import gradio as gr
+from langground import LangGround, text_palette
+
+
+state = {"loc_model": None, "llm_model": None, "model": None}
+
+
+def load_model(loc_model: str, llm_model: str) -> LangGround:
+    if (loc_model, llm_model) != (state["loc_model"], state["llm_model"]):
+        gr.Info("Loading models...", duration=5)
+        state.update({"model": LangGround(loc_model=loc_model, llm_model=llm_model), "loc_model": loc_model, "llm_model": llm_model})
+        gr.Info("Models loaded!", duration=2.5)
+    return state["model"]
+
+
+def predict(frame, question: str, threshold: float, loc_model: str, llm_model: str):
+    if not frame or not question.strip():
+        gr.Warning("Please provide both an image and a question")
+        return "", None, None
+
+    model = load_model(loc_model, llm_model)
+    return model.localize(frame, question, threshold=threshold)
+
+
+title = """
+<center> 
+
+<h1>  🔍 Language Localization </h1>
+<b> Upload an image and ask questions to find objects in it. <b>
+
+</center>
+"""
+
+
+with gr.Blocks() as demo:
+    gr.HTML(title)
+
+    with gr.Row():
+
+        with gr.Column(scale=1):
+            frame_input = gr.Image(type="pil", label="Upload Frame")
+
+        with gr.Column(scale=1):
+            with gr.Row():
+                with gr.Column(scale=1):
+
+                    loc_model_input = gr.Dropdown(
+                        choices=["yolo", "owl"],
+                        value="yolo",
+                        label="Localization Model",
+                    )
+                with gr.Column(scale=2):
+
+                    llm_model_input = gr.Dropdown(
+                        choices=[
+                            "Qwen/Qwen2.5-7B-Instruct",
+                            "OpenGVLab/InternVL2_5-8B",
+                            "OpenGVLab/InternVL2_5-4B",
+                            "OpenGVLab/InternVL2_5-2B",
+                            "OpenGVLab/InternVL2_5-1B",
+                        ],
+                        value="Qwen/Qwen2.5-7B-Instruct",
+                        label="LLM Model",
+                    )
+            threshold_input = gr.Slider(minimum=0, maximum=1, value=0.4, step=0.1, label="Threshold")
+            question_input = gr.Textbox(lines=2, placeholder="Enter your question here", label="Question")
+            objs = gr.Highlightedtext(show_legend=False, show_inline_category=False, color_map=text_palette, label="Objects Found")
+            submit_btn = gr.Button("Submit")
+
+    with gr.Row():
+        all_bbox_image = gr.Image(label="Found Objects")
+        llm_bbox_image = gr.Image(label="Selected Objects")
+
+    submit_btn.click(
+        fn=predict,
+        inputs=[frame_input, question_input, threshold_input, loc_model_input, llm_model_input],
+        outputs=[objs, all_bbox_image, llm_bbox_image],
+    )
+    examples = gr.Examples(
+        examples=[
+            ["assets/demo.jpeg", "I'm thirsty"],
+            ["assets/kitchen.webp", "The food has expired and is no longer safe to eat."],
+            ["assets/kitchen.webp", "The food is about to expire."],
+        ],
+        inputs=[frame_input, question_input],
+    )
+if __name__ == "__main__":
+    demo.launch()

example.py

@@ -0,0 +1,6 @@
+from langground import LangGround
+import cv2
+
+image = cv2.imread('./assets/demo.jpeg')
+lg = LangGround()
+lg.localize(image, "i'm thirsty")

langground/__init__.py

@@ -0,0 +1,2 @@
+from .orch import LangGround
+from .utils import text_palette

langground/llm.py

@@ -0,0 +1,90 @@
+import re
+import tenacity
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
+
+
+class LLM:
+    def __init__(self, model_id="Qwen/Qwen2.5-7B-Instruct",):
+
+        self.model_id = model_id
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        
+        # Load the model and tokenizer based on the model_id
+        if "meta-llama" in self.model_id:
+            self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+            self.model = AutoModelForCausalLM.from_pretrained(
+                model_id,
+                torch_dtype=torch.bfloat16,
+                device_map="auto"
+            )
+
+        elif "InternVL" in self.model_id:
+            self.model = AutoModel.from_pretrained(
+                model_id,
+                torch_dtype=torch.bfloat16,
+                low_cpu_mem_usage=True,
+                trust_remote_code=True,
+                device_map="auto"
+            ).eval()
+
+            self.tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, use_fast=False)
+
+        else:
+            self.model = AutoModelForCausalLM.from_pretrained(
+                model_id,
+                torch_dtype="auto",
+                device_map="auto"
+            )
+
+            self.tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    @torch.no_grad()
+    def generate(self, query):
+        if "meta-llama" in self.model_id:
+            messages = [
+                {"role": "user", "content": [
+                    {"type": "text", "text": f"{query}"}
+                ]}
+            ]
+            text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            model_inputs = self.tokenizer([text], return_tensors="pt").to(self.device)
+            generated_ids = self.model.generate(model_inputs.input_ids, max_new_tokens=512)
+            generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]
+
+            response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+        elif "InternVL" in self.model_id:
+            generation_config = dict(max_new_tokens=1024, do_sample=True)
+            response = self.model.chat(self.tokenizer, None, query, generation_config, history=None, return_history=False)
+        else:
+            messages = [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": query}]
+            text = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            model_inputs = self.tokenizer([text], return_tensors="pt").to(self.device)
+
+            generated_ids = self.model.generate(model_inputs.input_ids, max_new_tokens=512)
+            generated_ids = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]
+
+            response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+        return response
+
+    @tenacity.retry(stop=tenacity.stop_after_delay(10))
+    def answer(self, query, objects):
+        query = f"""
+        Extract the object that satisfies the intent of the query or determine the tool that aligns with the purpose of {query}.
+        pick the best option from the following: {', '.join(objects)},
+        Please return a list of all suitable options as long as they make sense in the format of a Python list in the following format: ```python\n['option1', 'option2', ...]```"""
+        res = self.generate(query)
+        match = re.search(r"`{3}python\\n(.*)`{3}", res, re.DOTALL)
+        if match:
+            res = match.group(1)
+            res = [r.translate(str.maketrans("", "", "_-")) for r in eval(res)]
+            return res
+        else:
+            # Try to extract content directly from brackets []
+            match_brackets = re.search(r"\[(.*?)\]", res, re.DOTALL)
+            if match_brackets:
+                res = match_brackets.group(0)  # Include brackets for eval
+                res = [r.translate(str.maketrans("", "", "_-")) for r in eval(res)]
+                return res
+            else:
+                raise ValueError(f"Failed to parse response: {res}")

langground/localizer.py

@@ -0,0 +1,60 @@
+from collections import defaultdict
+from pathlib import Path
+import torch
+from transformers import Owlv2Processor, Owlv2ForObjectDetection
+from PIL import Image
+from pathlib import Path
+from ultralytics import YOLO
+
+
+def build_localizer(model_name):
+    if model_name == "owl":
+        return OWL()
+    elif model_name == "yolo":
+        return YOLO11()
+    else:
+        raise ValueError(f"Unknown model name: {model_name}")
+
+
+class OWL:
+
+    def __init__(self):
+        model_name = "google/owlv2-large-patch14-ensemble"
+        self.processor = Owlv2Processor.from_pretrained(model_name)
+        self.model = Owlv2ForObjectDetection.from_pretrained(model_name).to("cuda")
+        self.model.eval()
+        self.objects_f = Path(__file__).parent / "objs" / "owl.txt"
+        self.objects = [line.strip() for line in self.objects_f.open().readlines()]
+        self.device = "cuda"
+
+    def localize(self, image, threshold=0.5):
+        image = Image.fromarray(image)
+        final = defaultdict(list)
+        with torch.inference_mode():
+            inputs = self.processor(text=self.objects, images=[image], return_tensors="pt").to(self.device)
+            outputs = self.model(**inputs)
+            target_sizes = torch.Tensor([image.size[::-1]]).to(self.device)
+            result = self.processor.post_process_object_detection(outputs=outputs, target_sizes=target_sizes, threshold=threshold)[0]
+
+            boxes, scores, labels = result["boxes"], result["scores"], result["labels"]
+            for box, score, label in zip(boxes, scores, labels):
+                final[self.objects[label]].append(box)
+        return final
+
+
+class YOLO11:
+    def __init__(self):
+        model_name = "yolo11m.pt"
+        self.model = YOLO(model_name)
+        self.objects_f = Path(__file__).parent / "objs" / "yolo.txt"
+        self.objects = [line.strip() for line in self.objects_f.open().readlines()]
+
+    def localize(self, image, threshold=0.5):
+        result = self.model(image, conf=threshold)[0]
+        boxes = result.boxes
+        bbox_ids = boxes.cls.cpu().numpy().astype(int)
+        boxes_xyxy = boxes.xyxy.cpu().numpy()
+        final = defaultdict(list)
+        for label, box in zip(bbox_ids, boxes_xyxy):
+            final[self.objects[label]].append(box)
+        return final

langground/objs/owl.txt

@@ -0,0 +1,1680 @@
+wine glass
+dumpster
+lip balm
+barge
+sock
+tricycle
+napkin
+bubble gum
+speed limit sign
+milestone
+headphones
+knocker (on a door)
+urn
+candy bar
+flash
+noddles
+mint candy
+nosebag (for animals)
+ladder
+boat
+chickpea
+razorblade
+keycard
+chisel
+coaster
+ice skate
+sweater
+onion
+gift wrap
+chocolate cake
+woodpecker
+bob
+serving tray
+flute
+prune
+thread
+potato
+cincture
+crowbar
+bottle cap
+can
+chain mail
+table teniis paddle
+boy
+cassette
+nutcracker
+electric drill
+typewriter
+fish (food)
+notebook
+duck
+studio couch
+kite
+baseball
+trench coat
+streetlight
+coffeemaker
+soup
+muffin
+humidifier
+soap
+dinning table
+egg tart
+fruit juice
+mashed potato
+target
+lifejacket
+bobby pin
+honeycomb
+harpsichord
+taxi
+refrigerator
+signboard
+hummus
+matchbox
+medicine
+cast
+carnivore
+frog
+newsstand
+french toast
+dishwasher
+file cabinet
+motor
+puffer (fish)
+apple
+water tower
+bonnet
+headlight
+trash bin can
+wheelchair
+roller skate
+pan (metal container)
+propeller
+baked goods
+hairnet
+tire
+smoothie
+milk
+recorder
+hinge
+winter melon
+hiking equipment
+bowling ball
+curtain
+cornbread
+coffeepot
+wallet/purse
+license plate
+baseball base
+pomegranate
+cabinet/shelf
+fishbowl
+carton
+boot
+thumbtack
+harmonica
+handle
+combination lock
+pea (food)
+wooden leg
+olive oil
+plant
+corded phone
+tea pot
+popsicle
+pineapple
+hummingbird
+porch
+kettle
+pennant
+statue (sculpture)
+mammal
+cleaning products
+sugar bowl
+limousine
+reamer (juicer)
+vacuum cleaner
+cooler (for food)
+ladle
+fire hydrant
+telephoto lens
+bird
+microphone
+pig
+game board
+squirrel
+trumpet
+punching bag
+notepad
+clutch bag
+billiard table
+hotair ballon
+wagon wheel
+ambulance
+maracas
+drink
+hoverboard
+steak
+cucumber
+hair spray
+fire truck
+scorpion
+violin
+turnip
+walking stick
+cupboard
+green bean
+crabmeat
+parachute
+shot glass
+sail
+skirt
+cayenne (spice)
+water faucet
+train (railroad vehicle)
+wristlet
+sushi
+forklift
+safety pin
+iron (for clothing)
+magnet
+camera lens
+fighter jet
+strawberry
+pouch
+skull
+bust
+bronze sculpture
+radish
+platter
+animal
+footstool
+toast (food)
+date (fruit)
+curling
+chandelier
+slow cooker
+goldfish
+handsaw
+nightshirt
+truffle (chocolate)
+saddle (on an animal)
+duct tape
+water gun
+fan
+cosmetics
+ladybug
+celery
+box
+birdbath
+hat
+lime
+person
+tinsel
+heart
+golf ball
+wine rack
+yoke (animal equipment)
+billards
+gravy boat
+hamster
+sharpie
+picture/frame
+booklet
+scooter
+nest
+arctic (type of shoe)
+swimwear
+flying disc
+ruler
+foal
+grater
+zebra
+glove
+elk
+garbage truck
+mushroon
+tortilla
+dairy
+poncho
+fudge
+coat hanger
+wet suit
+human leg
+hand towel
+elevator car
+tart
+croissant
+hairbrush
+beeper
+lion
+flap
+personal care
+cornice
+handkerchief
+bean curd
+cutting board
+extractor
+tree house
+headset
+seal
+toothbrush
+whisk
+fire extinguisher
+telephone
+antelope
+car
+cider
+pole
+gelatin
+measuring stick
+shoulder bag
+packet
+sparkler (fireworks)
+invertebrate
+deer
+hornet
+drone
+window blind
+horizontal bar
+fume hood
+barrow
+automatic washer
+trombone
+puncher
+hamper
+bathroom accessory
+penny (coin)
+mixer
+apricot
+inhaler
+turtleneck (clothing)
+computer monitor
+belt
+monkey
+bullet train
+tartan
+cube
+flute glass
+dispenser
+pickup truck
+rabbit
+icecream
+skullcap
+sunglasses
+boom microphone
+glasses
+oboe
+lollipop
+musical instrument
+bunk bed
+handcart
+flipper (footwear)
+motor vehicle
+shampoo
+hourglass
+road map
+sword
+walrus
+gourd
+glass (drink container)
+traffic cone
+cd
+ram (animal)
+teddy bear
+bomb
+mechanical fan
+water ski
+fedora
+barrel
+cosmetics brush/eyeliner pencil
+bull
+mouse
+carnation
+calendar
+pitcher
+clothes hamper
+buttefly
+armoire
+cleat (for securing rope)
+cougar
+electric chair
+tortoise
+mammoth
+tongs
+chips
+chaise longue
+webcam
+spider
+hand dryer
+canoe
+tank top (clothing)
+blender
+taillight
+leopard
+stool
+helicopter
+keg
+sausage
+wristband
+thermometer
+stepladder
+bolt
+rearview mirror
+axe
+baseball glove
+furniture
+carrot
+highchair
+rib (food)
+burrito
+passenger ship
+railcar (part of a train)
+short pants
+bowl
+earphone
+cabinet
+light switch
+washing machine/drying machine
+domestic ass
+golfcart
+ostrich
+mixing bowl
+pop (soda)
+flag
+toilet
+suv
+medal
+jeans
+chopstick
+ceiling fan
+tape measure
+reflector
+bulletin board
+turkey (food)
+mat (gym equipment)
+tow truck
+cruise ship
+sports uniform
+clothing
+control
+bench
+orange juice
+clippers (for plants)
+chopsticks
+table
+pressure cooker
+doughnut
+blackboard/whiteboard
+pepper mill
+trophy
+cowbell
+sandal (type of shoe)
+hamimelon
+corkboard
+file (tool)
+nailfile
+cab (taxi)
+head phone
+pinwheel
+briefcase
+birthday card
+barbell
+necklace
+baseball cap
+river boat
+tux
+book
+cistern
+cigar/cigarette
+tennis
+pencil sharpener
+tableware
+turkey
+string cheese
+musical keyboard
+flowerpot
+choker
+rag doll
+dropper
+deck chair
+horse
+bandanna
+lettuce
+steering wheel
+log
+snowplow
+television
+bamboo
+cork (bottle plug)
+cocktail shaker
+chinaware
+corset
+tripod
+reptile
+radio receiver
+peeler (tool for fruit and vegetables)
+diving board
+kitchen appliance
+banana
+calf
+coconut
+filing cabinet
+poker (fire stirring tool)
+birdcage
+green onion
+beanie
+birthday cake
+tea bag
+tarp
+lamb (animal)
+wedding ring
+baby buggy
+laptop
+tree
+palm tree
+wind chime
+segway
+step stool
+bicycle
+hockey stick
+pita (bread)
+crab
+saxophone
+gazelle
+bathtub
+robe
+brake light
+painting
+giraffe
+postbox (public)
+gull
+sunhat
+shaver (electric)
+melon
+bathrobe
+soccer ball
+barrel/bucket
+doorknob
+curling iron
+newspaper
+comb
+soup bowl
+dress hat
+scale (measuring instrument)
+human nose
+postcard
+convenience store
+pantyhose
+nail
+koala
+stationary bicycle
+chap
+seafood
+ski
+chicken
+knob
+rose
+gargoyle
+red panda
+toothpaste
+legging (clothing)
+bulldog
+amplifier
+colander
+drill
+suit (clothing)
+tap
+hair curler
+cover
+washbasin
+ratchet
+tablet
+cream pitcher
+dog bed
+subwoofer
+hamburger
+microwave
+lipstick
+picture frame
+scoreboard
+centipede
+asparagus
+dog
+building
+spatula
+cone
+dental floss
+container
+medical equipment
+funnel
+car (automobile)
+training bench
+pocket watch
+megaphone
+bathroom cabinet
+thermos bottle
+crucifix
+shotgun
+insect
+rhinoceros
+popcorn
+crisp (potato chip)
+kitchen sink
+wrench
+bedpan
+cricket ball
+folding chair
+coin
+rubber band
+minivan
+sherbert
+bakset
+cutting/chopping board
+cigarette
+woman
+suit
+pliers
+yogurt
+crape
+beef (food)
+cream
+jumpsuit
+cornet
+steak (food)
+gameboard
+bath mat
+bidet
+door handle
+desk
+campel
+common fig
+wedding cake
+bib
+kitchen knife
+basketball backboard
+skating and skiing shoes
+swimsuit
+tong
+chime
+thermostat
+tablet computer
+passport
+cowboy hat
+whipped cream
+frisbee
+basket
+blazer
+lemonade
+ship
+bouquet
+stretcher
+red cabbage
+other balls
+shaving cream
+battery
+piano
+peanut butter
+gorilla
+falcon
+fashion accessory
+buoy
+ski boot
+ballon
+pendulum
+windshield wiper
+drinking straw
+gargle
+pen/pencil
+eggbeater
+artichoke
+scallop
+cattle
+high heels
+strainer
+race car
+formula 1
+yak
+earplug
+noseband (for animals)
+nuts
+toiletry
+bow tie
+bread
+juice
+faucet
+pizza
+dollhouse
+air conditioner
+nightstand
+overalls (clothing)
+cello
+chair
+easel
+swan
+footwear
+paper towel
+bow and arrow
+chicken (animal)
+futon
+router (computer equipment)
+sofa bed
+orange/tangerine
+salt and pepper shakers
+strap
+toaster
+pew (church bench)
+worm
+beaker
+wolf
+sneakers
+map
+jar
+stirrup
+ferry
+luggage and bags
+canteen
+toilet paper
+candle holder
+blackboard
+cheetah
+card
+batter (food)
+thimble
+bat
+lighthouse
+slipper (footwear)
+cooker
+brassiere
+starfish
+human head
+coleslaw
+parchment
+shoe
+bus (vehicle)
+polo shirt
+pigeon
+mule
+ice cream
+lizard
+parking meter
+fast food
+grinder
+soya milk
+pitchfork
+gasmask
+pitcher (vessel for liquid)
+human beard
+visor
+telephone pole
+jewelry
+chocolate milk
+salami
+candy cane
+bicycle wheel
+grill
+waffle iron
+shelf
+jean
+grapefruit
+pipe bowl
+bell pepper
+human foot
+perfume
+carriage
+fruit
+checkbook
+orange
+tequila
+hatbox
+magpie
+cornmeal
+police cruiser
+stroller
+joystick
+neckerchief
+computer keyboard
+swing
+compass
+bullhorn
+chili (vegetable)
+alligator
+silo
+shirt
+corn
+shower cap
+racket
+camel
+flower
+cocoa (beverage)
+dollar
+mailbox (at home)
+tool
+lamp
+mousepad
+spectacles
+handgun
+lifesaver
+maple
+sink
+pan (for cooking)
+rocking chair
+vulture
+sandal
+school bus
+cap (headwear)
+scarf
+handbag/satchel
+convertible (automobile)
+igniter
+rays and skates
+isopod
+dinosaur
+hardback book
+telephone booth
+cosmetics mirror
+cleansing agent
+rice
+power shovel
+projector
+crutch
+toolbox
+puffin
+cloak
+mandarin orange
+windmill
+softball
+checkerboard
+squid
+armor
+treadmill
+vinegar
+puppy
+paintbrush
+wrench
+tie
+watch
+weathervane
+urinal
+fork
+turban
+headstall (for horses)
+rollerblade
+volleyball
+salsa
+boots
+ring binder
+kilt
+black sheep
+ice pack
+clothespin
+flamingo
+water cooler
+fig (fruit)
+stylus
+tape (sticky cloth or paper)
+crown
+cupcake
+raincoat
+cracker
+cocktail
+ham
+tray
+tambourine
+lightning rod
+missile
+bowl/basin
+towel rack
+crib
+cabin car
+can opener
+magazine
+hair dryer
+tin can
+human ear
+dinghy
+human hand
+zucchini
+ice maker
+van
+plumbing fixture
+bell
+sombrero
+mound (baseball)
+water jug
+egg roll
+notepaper
+fox
+remote control
+straw (for drinking)
+egg yolk
+quiche
+ginger
+motor scooter
+raccoon
+toothpick
+measuring cup
+cape
+rocket
+washing machine
+scissors
+shawl
+food
+sailboat
+street sign
+canned
+clip
+lobster
+coil
+stew
+waste container
+duckling
+hippopotamus
+bobbin
+squash
+cannon
+jug
+cauliflower
+timer
+honey
+whiteboard
+baguet
+pirate flag
+dog collar
+castle
+butter
+goat
+brownie
+television set
+solar array
+paper plate
+ashtray
+lampshade
+trash can
+cake
+paint brush
+dumbbell
+armchair
+fish
+ironing board
+deadbolt
+awning
+banjo
+coffee cup
+eagle
+bridal gown
+sun hat
+girl
+shorts
+crate
+root beer
+elephant
+bracelet
+computer mouse
+table tennis racket
+apron
+airplane
+hookah
+beer can
+green vegetables
+tractor (farm equipment)
+flower arrangement
+bun
+showerhead
+band-aid
+bagel
+pancake
+countertop
+fleece
+human hair
+digital clock
+anklet
+dumpling
+kiwi fruit
+vending machine
+sandwich
+cub (animal)
+manger
+skateboard
+beachball
+watercraft
+salmon (fish)
+bolo tie
+shredder (for paper)
+die
+bow (weapon)
+trophy cup
+hot-air balloon
+sweat pants
+adhesive tape
+atomizer
+parrot
+suspenders
+mallard
+salmon (food)
+paper cutter
+saucer
+moths and butterflies
+snake
+cabinetry
+board eraser
+brass plaque
+wooden spoon
+tennis racket
+vehicle registration plate
+shepherd dog
+boiled egg
+soupspoon
+tinfoil
+chocolate mousse
+mop
+hot sauce
+wallet
+projectile (weapon)
+otter
+egg
+kitten
+cantaloupe
+mouse (computer equipment)
+pin (non jewelry)
+fire alarm
+brush
+lego
+waffle
+land vehicle
+wok
+tissue
+bass horn
+gag
+mascot
+tennis ball
+towel
+antenna
+pajamas
+raven
+shield
+eggplant
+mast
+bandage
+cooking spray
+drum
+alarm clock
+martini
+cabbage
+saucepan
+pen
+auto part
+lanyard
+wig
+syringe
+vegetable
+shopping cart
+avocado
+pumpkin
+wardrobe
+puppet
+palette
+wall socket
+chopping board
+pretzel
+window box (for plants)
+fountain
+american football
+pepper
+bucket
+harmonium
+brown bear
+marker
+salad plate
+underdrawers
+eel
+earring
+parakeet
+blue jay
+mango
+accordion
+coffee table
+fireplace
+mallet
+clock tower
+passenger car (part of a train)
+stuffed toy
+bagpipe
+infant bed
+key
+aircraft
+lynx
+jacuzzi
+cookie
+dish
+cigar box
+induction cooker
+jacket
+trailer truck
+stirrer
+hammer
+wine bottle
+basketball
+stove
+caterpillar
+human body
+horse carriage
+garlic
+door
+ant
+pudding
+penguin
+human mouth
+kitchenware
+sleeping bag
+eraser
+drumstick
+dining table
+playpen
+picnic basket
+knee pad
+beer
+microwave oven
+almond
+crocodile
+cymbal
+banner
+houseboat
+jelly bean
+cheese
+power plugs and sockets
+houseplant
+padlock
+runner (carpet)
+office supplies
+vase
+canary
+umbrella
+crescent roll
+tower
+snail
+radiator
+blinker
+skewer
+dress
+gun
+toy
+tablecloth
+meat ball
+gas stove
+truck
+pizza cutter
+dress suit
+latch
+gemstone
+wild bird
+other fish
+baseball bat
+garden hose
+trousers
+place mat
+applesauce
+ballet skirt
+remote
+traffic sign
+envelope
+billboard
+flask
+octopus (animal)
+poker card
+boxing glove
+binoculars
+crock pot
+paperback book
+swim cap
+bulletproof vest
+duffel bag
+cooking utensil
+crumb
+bookcase
+headboard
+nut
+hammock
+pillow
+baboon
+tachometer
+tiara
+snowman
+tag
+marine mammal
+teakettle
+parasail (sports)
+pastry
+tent
+golf club
+ferret
+salad
+power outlet
+blimp
+coat
+sports car
+candle
+bow (decorative ribbons)
+broom
+snack
+flagpole
+side table
+trampoline
+bowler hat
+indoor rower
+vehicle
+shellfish
+table tennis
+cargo ship
+pencil
+freshener
+heron
+fire engine
+persimmon
+pocketknife
+jet ski
+casserole
+ottoman
+machinery vehicle
+coloring material
+mobile phone
+goose
+tape
+record player
+bed
+phonograph record
+bible
+pug-dog
+spotlight
+chessboard
+bow-tie
+griddle
+armband
+cart
+leather shoes
+cat
+cock
+snowboard
+loveseat
+rat
+tick
+prawn
+walking cane
+teacup
+bead
+marine invertebrates
+handcuff
+bicycle helmet
+teapot
+table lamp
+canister
+rugby ball
+surveillance camera
+pacifier
+french fries
+cockroach
+comic book
+detergent
+cookies
+patty (food)
+ping-pong ball
+phonebook
+shopping bag
+cigarette case
+donkey
+coatrack
+coffee
+oil lamp
+raspberry
+stairs
+water bottle
+shaker
+cherry
+grits
+rifle
+poster
+cellular telephone
+ski parka
+surfboard
+pelican
+saddle blanket
+microscope
+willow
+blanket
+organ
+sweet potato
+wineglass
+oyster
+cushion
+radar
+whistle
+blouse
+lightbulb
+soap dispenser
+pie
+shower head
+dalmatian
+milk can
+gloves
+handbag
+seaplane
+scale
+fireplug
+crayon
+halter top
+award
+sea lion
+calculator
+street lights
+camper (vehicle)
+tapestry
+traffic light
+pencil case
+plume
+jam
+bait
+vat
+pet
+first-aid kit
+tobacco pipe
+sportswear
+bus
+parasol
+sponge
+french
+recliner
+wheel
+bottle opener
+armadillo
+shrimp
+bat (animal)
+aquarium
+knitting needle
+dartboard
+jellyfish
+milkshake
+cantaloup
+cell phone
+chalice
+dove
+life jacket
+dirt bike
+horse buggy
+baozi
+watermelon
+crosswalk sign
+edible corn
+pot
+mitten
+polar bear
+clock
+chocolate bar
+kitchen & dining room table
+papaya
+flip-flop (sandal)
+gondola (boat)
+wine bucket
+converter
+mug
+butterfly
+dresser
+pencil box
+army tank
+cash register
+rolling pin
+parka
+scrubbing brush
+tea
+bulldozer
+carpet
+vodka
+donut
+human face
+whale
+jeep
+lemon
+office building
+football (american)
+breechcloth
+stereo (sound system)
+crane
+aerosol can
+pasta
+green beans
+vent
+speaker (stero equipment)
+jaguar
+facial tissue holder
+blueberry
+wood-burning stove
+ball
+generator
+lab coat
+beer bottle
+necktie
+shark
+lavender
+blackberry
+hurdle
+bedspread
+plum
+keyboard
+grape
+cylinder
+roller skates
+tabasco sauce
+home plate (baseball)
+guitar
+rickshaw
+sports equipment
+extention cord
+orange (fruit)
+bottle
+human eye
+trunk
+skyscraper
+needle
+jewel
+shower
+sandals
+pool table
+man
+closet
+gravestone
+coverall
+sweatband
+horn
+globe
+seahorse
+helmet
+kitchen table
+balloon
+music stool
+lasagna
+guacamole
+spice rack
+legume
+sling (bandage)
+diaper
+coffee maker
+lamppost
+pinecone
+gondola
+owl
+luggage
+diary
+dessert
+okra
+crow
+tomato
+peach
+bear
+bee
+sewing machine
+shower curtain
+birdfeeder
+sweatshirt
+table-tennis table
+submarine
+water heater
+oar
+drawer
+balance beam
+slippers
+liquor
+dishtowel
+stapler (stapling machine)
+jersey
+wreath
+tape measur/ ruler
+trolley
+light bulb
+lantern
+raft
+identity card
+car battery
+sawhorse
+coffee machine
+camera
+triangle (musical instrument)
+oven
+rodent
+durian
+weapon
+doormat
+belt buckle
+figurine
+chest of drawers
+heavy truck
+storage box
+skiboard
+submarine sandwich
+tank
+mirror
+pickle
+ski pole
+bath towel
+wine
+window
+golf cart
+underwear
+water scooter
+stethoscope
+sea turtle
+alpaca
+seashell
+hanger
+alcohol
+clipboard
+sled
+plastic bag
+other shoes
+pipe
+dishwasher detergent
+chainsaw
+fire hose
+tuba
+manatee
+pottery
+camcorder
+swimming pool
+couch
+scarecrow
+bowling equipment
+lighter
+heater
+kayak
+leather
+folder
+lily
+flannel
+scraper
+crawfish
+knife
+ipod
+slide
+pistol
+tiger
+freight car
+fishing rod
+mattress
+kimono
+speaker
+costume
+rice cooker
+watering can
+goggles
+drum (musical instrument)
+pear
+wall clock
+sour cream
+pegboard
+masher
+hook
+printer
+sugarcane (plant)
+torch
+blinder (for horses)
+cufflink
+vest
+dishrag
+headscarf
+earrings
+brussels sprouts
+dragonfly
+porcupine
+hand glass
+mushroom
+cake stand
+christmas tree
+sparrow
+lamb-chop
+machine gun
+motorcycle
+unicycle
+broccoli
+saltshaker
+saddlebag
+hedgehog
+grocery bag
+pad
+seat belt
+squid (food)
+panda
+computer box
+spring rolls
+soccer
+miniskirt
+stop sign
+potted plant
+kennel
+stagecoach
+dustpan
+lawn mower
+giant panda
+beanbag
+mail slot
+satchel
+street light
+suitcase
+seabird
+beetle
+corkscrew
+crab (animal)
+escargot
+steak knife
+binder
+tights (clothing)
+food processor
+plate
+quilt
+button
+cow
+shovel
+plow (farm equipment)
+cat furniture
+toaster oven
+mask
+barrette
+stapler
+paddle
+jet plane
+mixer (kitchen tool)
+train
+locker
+ferris wheel
+television camera
+birdhouse
+flashlight
+grizzly
+dagger
+tassel
+sheep
+fax
+condiment
+cappuccino
+dixie cup
+candy
+sculpture
+kitchen utensil
+crossbar
+sofa
+quesadilla
+beret
+cue
+bread-bin
+frying pan
+beehive
+clasp
+spear
+turtle
+football
+taco
+clementine
+ring
+doll
+tank (storage vessel)
+space shuttle
+manhole
+octopus (food)
+hotplate
+eclair
+videotape
+kangaroo
+piggy bank
+sunflower
+pony
+paperweight
+cup
+band aid
+clarinet
+garbage
+hog
+yacht
+meatball
+inkpad
+eyepatch
+face powder
+house
+hose
+tote bag
+cardigan
+veil
+wagon
+headband
+harbor seal
+cassette deck
+snowmobile
+screwdriver
+cd player
+dice
+skunk
+omelet
+router/modem
+bookmark
+backpack
+cabana
+laptop computer
+poker chip
+life buoy
+potholder
+human arm
+football helmet
+broach
+sharpener
+hot dog
+harp
+hairpin
+windsock
+crouton
+receipt
+business card
+dolphin
+spoon
+shears
+dish antenna
+money
+home appliance
+ax

langground/objs/yolo.txt

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light 
+fire hydrant 
+stop sign 
+parking meter 
+bench 
+bird 
+cat 
+dog 
+horse 
+sheep 
+cow 
+elephant 
+bear 
+zebra 
+giraffe 
+backpack 
+umbrella 
+handbag 
+tie 
+suitcase 
+frisbee 
+skis 
+snowboard 
+sports ball 
+kite 
+baseball bat 
+baseball glove 
+skateboard 
+surfboard 
+tennis racket 
+bottle 
+wine glass 
+cup 
+fork 
+knife 
+spoon 
+bowl 
+banana 
+apple 
+sandwich 
+orange 
+broccoli 
+carrot 
+hot dog 
+pizza 
+donut 
+cake 
+chair 
+couch 
+potted plant 
+bed 
+dining table 
+toilet 
+tv 
+laptop 
+mouse 
+remote 
+keyboard 
+cell phone 
+microwave 
+oven 
+toaster 
+sink 
+refrigerator 
+book 
+clock 
+vase 
+scissors 
+teddy bear 
+hair drier 
+toothbrush

langground/orch.py

@@ -0,0 +1,22 @@
+from .localizer import build_localizer
+from .llm import LLM
+from .utils import image_w_box
+import numpy as np
+
+class LangGround:
+
+    def __init__(self, loc_model="owl", llm_model="Qwen/Qwen2.5-7B-Instruct"):
+
+        self.loc = build_localizer(loc_model)
+        self.llm = LLM(llm_model)
+
+    def localize(self, frame, question, **kwargs):
+
+        frame = np.array(frame)
+        objxbox = self.loc.localize(frame, kwargs.get("threshold", 0.5))
+        locobjs = self.llm.answer(question, objxbox.keys())
+        locobjxbox = {k: v for k, v in objxbox.items() if k in locobjs}
+        all_box_image = image_w_box(frame, objxbox)
+        llm_box_image = image_w_box(frame, locobjxbox)
+        texts = [(text, str(idx)) for idx, text in enumerate(locobjs)]
+        return texts, all_box_image, llm_box_image

langground/utils.py

@@ -0,0 +1,117 @@
+from PIL import Image
+import supervision as sv
+import numpy as np
+from torch import tensor
+import cv2
+
+colors = sv.ColorPalette.from_hex(
+    [
+        "#a1c9f4",
+        "#ffb482",
+        "#8de5a1",
+        "#ff9f9b",
+        "#d0bbff",
+        "#debb9b",
+        "#fab0e4",
+        "#cfcfcf",
+        "#fffea3",
+        "#b9f2f0",
+        "#a1c9f4",
+        "#ffb482",
+        "#8de5a1",
+        "#ff9f9b",
+        "#d0bbff",
+        "#debb9b",
+        "#fab0e4",
+        "#cfcfcf",
+        "#fffea3",
+        "#b9f2f0",
+    ]
+)
+
+text_palette = {str(idx): colors.by_idx(idx).as_hex() for idx in range(50)}
+
+
+def image_w_box(image,objxbox):
+
+    box_annotator = sv.BoxCornerAnnotator(thickness=10, corner_length=30, color=colors)
+    label_annotator = sv.LabelAnnotator(color=colors)
+    mask_annotator = sv.MaskAnnotator(opacity=0.2, color=colors)
+
+    xyxys = np.array([v.tolist() for boxes in objxbox.values() for v in boxes])
+    unique_labels = sorted(objxbox.keys())
+    class_id_map = dict(enumerate(unique_labels))
+    labels = [l for l, boxes in objxbox.items() for _ in boxes]
+    class_id = [list(class_id_map.values()).index(label) for label in labels]
+
+    masks = np.zeros((len(xyxys), image.shape[0], image.shape[1]), dtype=bool)
+    for i, (x1, y1, x2, y2) in enumerate(xyxys):
+        masks[i, int(y1):int(y2), int(x1):int(x2)] = labels[i]
+
+    if len(xyxys) == 0:
+        return image
+    detections = sv.Detections(
+        xyxy=xyxys,
+        mask=masks,
+        class_id=np.array(class_id),
+    )
+    # Convert RGB to BGR for annotation
+    image_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    # After annotation, convert back to RGB
+    annotated_image = box_annotator.annotate(scene=image_bgr.copy(), detections=detections)
+    annotated_image = label_annotator.annotate(scene=annotated_image, detections=detections, labels=labels)
+    annotated_image = mask_annotator.annotate(scene=annotated_image, detections=detections)
+
+    return cv2.cvtColor(annotated_image, cv2.COLOR_BGR2RGB)
+
+
+def image_w_box_cv2(image, objxbox):
+    if not isinstance(image, np.ndarray):
+        raise ValueError("Input image must be a NumPy array.")
+
+    image_copy = image.copy()
+
+    font = cv2.FONT_HERSHEY_SIMPLEX
+
+    height, width, _ = image.shape
+    font_scale = max(0.5, min(width, height) / 1000)  
+    font_thickness = max(1, int(font_scale * 2))  
+
+    for label, boxes in objxbox.items():
+        for box in boxes:
+            print("box", box)
+
+            x1, y1, x2, y2 = map(int, box.tolist())
+
+            cv2.rectangle(image_copy, (x1, y1), (x2, y2), color=(0, 0, 255), thickness=2)
+
+            label_text = f"{label}"
+
+            (text_width, text_height), baseline = cv2.getTextSize(
+                label_text, font, font_scale, font_thickness
+            )
+
+            text_x1 = x1
+            text_y1 = y1 - text_height - baseline
+            text_x2 = x1 + text_width
+            text_y2 = y1
+
+            cv2.rectangle(image_copy, (text_x1, text_y1), (text_x2, text_y2), color=(255, 255, 255), thickness=-1)
+
+            cv2.putText(
+                image_copy,
+                label_text,
+                (x1, y1 - baseline),
+                font,
+                font_scale,
+                color=(0, 0, 255),
+                thickness=font_thickness,
+                lineType=cv2.LINE_AA,
+            )
+
+    return image_copy
+
+if __name__ == '__main__':
+    image = Image.open("assets/demo.jpeg")
+    objxbox = {'computer monitor': [tensor([ 169.5367,  301.8970, 3045.2866, 2145.4736], device='cuda:0')], 'lamp': [tensor([3400.5979,  981.1383, 4102.7178, 2417.0103], device='cuda:0')], 'kettle': [tensor([4435.6953, 1981.3882, 5318.8530, 2972.8535], device='cuda:0')], 'table': [tensor([3108.2896, 2602.6494, 5795.3037, 4201.5000], device='cuda:0')], 'business card': [tensor([ 751.5681, 2817.4629,  945.1781, 2976.9883], device='cuda:0')], 'dog': [tensor([2155.5217, 2504.7114, 2562.2791, 3173.9731], device='cuda:0'), tensor([1013.7704, 2669.0864, 1560.3319, 3452.0579], device='cuda:0')], 'inkpad': [tensor([ 755.5402, 2983.9380,  962.8440, 3176.2158], device='cuda:0')], 'mouse': [tensor([2752.5286, 3038.9062, 3046.8740, 3297.1704], device='cuda:0')], 'tray': [tensor([3314.1667, 2722.6509, 4805.7476, 3684.2314], device='cuda:0')], 'computer keyboard': [tensor([ 203.7615, 2907.8442,  737.0474, 3416.8616], device='cuda:0')], 'laptop': [tensor([ 525.8097, 2439.1343, 2882.1917, 4261.9614], device='cuda:0')], 'keyboard': [tensor([ 659.9836, 3511.1763, 2828.9368, 4271.0059], device='cuda:0')], 'cookie': [tensor([4638.1128, 3625.8831, 5082.5796, 4013.4021], device='cuda:0')]}
+    image_w_box(image, objxbox).show()

pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "langground"
+version = "0.1.0"
+description = "Use natural language to ground relevant things."
+readme = "README.md"
+authors = [
+    {name = "Jing Bi", email = "jbi5@ur.rochester.edu"},
+    {name = "Guangyu Sun", email = "guangyu@ucf.edu"}
+]
+requires-python = ">=3.8"
+dynamic = ["dependencies"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["langground", "langground.*"]

requirements.txt

@@ -0,0 +1,12 @@
+torch
+transformers
+opencv-python
+tenacity
+accelerate
+pillow
+scipy
+gradio
+supervision
+ultralytics
+einops
+timm