Spaces:
Nymbo
/
Runtime error

badayvedat commited on
Commit
a824a18
1 Parent(s): fc24801

feat: Add LLaVA model

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +31 -0
  2. LICENSE +201 -0
  3. README.md +5 -10
  4. app.py +600 -0
  5. data/prompts/complex_reasoning/000_caps.txt +18 -0
  6. data/prompts/complex_reasoning/000_conv.txt +5 -0
  7. data/prompts/complex_reasoning/001_caps.txt +18 -0
  8. data/prompts/complex_reasoning/001_conv.txt +5 -0
  9. data/prompts/complex_reasoning/002_caps.txt +7 -0
  10. data/prompts/complex_reasoning/002_conv.txt +5 -0
  11. data/prompts/complex_reasoning/system_message.txt +10 -0
  12. data/prompts/conversation/000_caps.txt +5 -0
  13. data/prompts/conversation/000_conv.txt +29 -0
  14. data/prompts/conversation/001_caps.txt +5 -0
  15. data/prompts/conversation/001_conv.txt +37 -0
  16. data/prompts/conversation/system_message.txt +12 -0
  17. data/prompts/detail_description/000_caps.txt +18 -0
  18. data/prompts/detail_description/000_conv.txt +3 -0
  19. data/prompts/detail_description/001_caps.txt +18 -0
  20. data/prompts/detail_description/001_conv.txt +5 -0
  21. data/prompts/detail_description/002_caps.txt +15 -0
  22. data/prompts/detail_description/002_conv.txt +3 -0
  23. data/prompts/detail_description/system_message.txt +7 -0
  24. docs/Customize_Component.md +20 -0
  25. docs/Data.md +29 -0
  26. docs/LLaVA_Bench.md +31 -0
  27. docs/LLaVA_from_LLaMA2.md +29 -0
  28. docs/LoRA.md +46 -0
  29. docs/MODEL_ZOO.md +136 -0
  30. docs/ScienceQA.md +53 -0
  31. examples/extreme_ironing.jpg +0 -0
  32. examples/waterview.jpg +0 -0
  33. llava/__init__.py +1 -0
  34. llava/constants.py +12 -0
  35. llava/conversation.py +381 -0
  36. llava/eval/eval_gpt_review.py +113 -0
  37. llava/eval/eval_gpt_review_bench.py +121 -0
  38. llava/eval/eval_gpt_review_visual.py +118 -0
  39. llava/eval/eval_science_qa.py +99 -0
  40. llava/eval/eval_science_qa_gpt4.py +104 -0
  41. llava/eval/eval_science_qa_gpt4_requery.py +149 -0
  42. llava/eval/generate_webpage_data_from_table.py +111 -0
  43. llava/eval/model_qa.py +85 -0
  44. llava/eval/model_vqa.py +112 -0
  45. llava/eval/model_vqa_science.py +141 -0
  46. llava/eval/qa_baseline_gpt35.py +74 -0
  47. llava/eval/run_llava.py +97 -0
  48. llava/eval/summarize_gpt_review.py +50 -0
  49. llava/eval/webpage/figures/alpaca.png +0 -0
  50. llava/eval/webpage/figures/bard.jpg +0 -0
.gitignore ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python
2
+ __pycache__
3
+ *.pyc
4
+ *.egg-info
5
+ dist
6
+
7
+ # Log
8
+ *.log
9
+ *.log.*
10
+ *.json
11
+ *.jsonl
12
+
13
+ # Data
14
+ !**/alpaca-data-conversation.json
15
+
16
+ # Editor
17
+ .idea
18
+ *.swp
19
+
20
+ # Other
21
+ .DS_Store
22
+ wandb
23
+ output
24
+
25
+ checkpoints
26
+ ckpts*
27
+
28
+ .ipynb_checkpoints
29
+ *.ipynb
30
+
31
+ *.log
LICENSE ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "[]"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright [yyyy] [name of copyright owner]
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
README.md CHANGED
@@ -1,13 +1,8 @@
1
  ---
2
  title: LLaVA
3
- emoji: 🏃
4
- colorFrom: gray
5
- colorTo: pink
6
  sdk: gradio
7
- sdk_version: 3.47.1
8
- app_file: app.py
9
- pinned: false
10
- license: apache-2.0
11
- ---
12
-
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
  title: LLaVA
3
+ emoji: 🔥
4
+ colorFrom: purple
5
+ colorTo: gray
6
  sdk: gradio
7
+ app_port: 7860
8
+ ---
 
 
 
 
 
app.py ADDED
@@ -0,0 +1,600 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import datetime
3
+ import hashlib
4
+ import json
5
+ import os
6
+ import subprocess
7
+ import sys
8
+ import time
9
+
10
+ import gradio as gr
11
+ import requests
12
+
13
+ from llava.constants import LOGDIR
14
+ from llava.conversation import SeparatorStyle, conv_templates, default_conversation
15
+ from llava.utils import (
16
+ build_logger,
17
+ moderation_msg,
18
+ server_error_msg,
19
+ violates_moderation,
20
+ )
21
+
22
+ logger = build_logger("gradio_web_server", "gradio_web_server.log")
23
+
24
+ headers = {"User-Agent": "LLaVA Client"}
25
+
26
+ no_change_btn = gr.Button.update()
27
+ enable_btn = gr.Button.update(interactive=True)
28
+ disable_btn = gr.Button.update(interactive=False)
29
+
30
+ priority = {
31
+ "vicuna-13b": "aaaaaaa",
32
+ "koala-13b": "aaaaaab",
33
+ }
34
+
35
+
36
+ def get_conv_log_filename():
37
+ t = datetime.datetime.now()
38
+ name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
39
+ return name
40
+
41
+
42
+ def get_model_list():
43
+ ret = requests.post(args.controller_url + "/refresh_all_workers")
44
+ assert ret.status_code == 200
45
+ ret = requests.post(args.controller_url + "/list_models")
46
+ models = ret.json()["models"]
47
+ models.sort(key=lambda x: priority.get(x, x))
48
+ logger.info(f"Models: {models}")
49
+ return models
50
+
51
+
52
+ get_window_url_params = """
53
+ function() {
54
+ const params = new URLSearchParams(window.location.search);
55
+ url_params = Object.fromEntries(params);
56
+ console.log(url_params);
57
+ return url_params;
58
+ }
59
+ """
60
+
61
+
62
+ def load_demo(url_params, request: gr.Request):
63
+ logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")
64
+
65
+ dropdown_update = gr.Dropdown.update(visible=True)
66
+ if "model" in url_params:
67
+ model = url_params["model"]
68
+ if model in models:
69
+ dropdown_update = gr.Dropdown.update(value=model, visible=True)
70
+
71
+ state = default_conversation.copy()
72
+ return state, dropdown_update
73
+
74
+
75
+ def load_demo_refresh_model_list(request: gr.Request):
76
+ logger.info(f"load_demo. ip: {request.client.host}")
77
+ models = get_model_list()
78
+ state = default_conversation.copy()
79
+ dropdown_update = gr.Dropdown.update(
80
+ choices=models, value=models[0] if len(models) > 0 else ""
81
+ )
82
+ return state, dropdown_update
83
+
84
+
85
+ def vote_last_response(state, vote_type, model_selector, request: gr.Request):
86
+ with open(get_conv_log_filename(), "a") as fout:
87
+ data = {
88
+ "tstamp": round(time.time(), 4),
89
+ "type": vote_type,
90
+ "model": model_selector,
91
+ "state": state.dict(),
92
+ "ip": request.client.host,
93
+ }
94
+ fout.write(json.dumps(data) + "\n")
95
+
96
+
97
+ def upvote_last_response(state, model_selector, request: gr.Request):
98
+ logger.info(f"upvote. ip: {request.client.host}")
99
+ vote_last_response(state, "upvote", model_selector, request)
100
+ return ("",) + (disable_btn,) * 3
101
+
102
+
103
+ def downvote_last_response(state, model_selector, request: gr.Request):
104
+ logger.info(f"downvote. ip: {request.client.host}")
105
+ vote_last_response(state, "downvote", model_selector, request)
106
+ return ("",) + (disable_btn,) * 3
107
+
108
+
109
+ def flag_last_response(state, model_selector, request: gr.Request):
110
+ logger.info(f"flag. ip: {request.client.host}")
111
+ vote_last_response(state, "flag", model_selector, request)
112
+ return ("",) + (disable_btn,) * 3
113
+
114
+
115
+ def regenerate(state, image_process_mode, request: gr.Request):
116
+ logger.info(f"regenerate. ip: {request.client.host}")
117
+ state.messages[-1][-1] = None
118
+ prev_human_msg = state.messages[-2]
119
+ if type(prev_human_msg[1]) in (tuple, list):
120
+ prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
121
+ state.skip_next = False
122
+ return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
123
+
124
+
125
+ def clear_history(request: gr.Request):
126
+ logger.info(f"clear_history. ip: {request.client.host}")
127
+ state = default_conversation.copy()
128
+ return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
129
+
130
+
131
+ def add_text(state, text, image, image_process_mode, request: gr.Request):
132
+ logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}")
133
+ if len(text) <= 0 and image is None:
134
+ state.skip_next = True
135
+ return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
136
+ if args.moderate:
137
+ flagged = violates_moderation(text)
138
+ if flagged:
139
+ state.skip_next = True
140
+ return (state, state.to_gradio_chatbot(), moderation_msg, None) + (
141
+ no_change_btn,
142
+ ) * 5
143
+
144
+ text = text[:1536] # Hard cut-off
145
+ if image is not None:
146
+ text = text[:1200] # Hard cut-off for images
147
+ if "<image>" not in text:
148
+ # text = '<Image><image></Image>' + text
149
+ text = text + "\n<image>"
150
+ text = (text, image, image_process_mode)
151
+ if len(state.get_images(return_pil=True)) > 0:
152
+ state = default_conversation.copy()
153
+ state.append_message(state.roles[0], text)
154
+ state.append_message(state.roles[1], None)
155
+ state.skip_next = False
156
+ return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
157
+
158
+
159
+ def http_bot(
160
+ state, model_selector, temperature, top_p, max_new_tokens, request: gr.Request
161
+ ):
162
+ logger.info(f"http_bot. ip: {request.client.host}")
163
+ start_tstamp = time.time()
164
+ model_name = model_selector
165
+
166
+ if state.skip_next:
167
+ # This generate call is skipped due to invalid inputs
168
+ yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
169
+ return
170
+
171
+ if len(state.messages) == state.offset + 2:
172
+ # First round of conversation
173
+ if "llava" in model_name.lower():
174
+ if "llama-2" in model_name.lower():
175
+ template_name = "llava_llama_2"
176
+ elif "v1" in model_name.lower():
177
+ if "mmtag" in model_name.lower():
178
+ template_name = "v1_mmtag"
179
+ elif (
180
+ "plain" in model_name.lower()
181
+ and "finetune" not in model_name.lower()
182
+ ):
183
+ template_name = "v1_mmtag"
184
+ else:
185
+ template_name = "llava_v1"
186
+ elif "mpt" in model_name.lower():
187
+ template_name = "mpt"
188
+ else:
189
+ if "mmtag" in model_name.lower():
190
+ template_name = "v0_mmtag"
191
+ elif (
192
+ "plain" in model_name.lower()
193
+ and "finetune" not in model_name.lower()
194
+ ):
195
+ template_name = "v0_mmtag"
196
+ else:
197
+ template_name = "llava_v0"
198
+ elif "mpt" in model_name:
199
+ template_name = "mpt_text"
200
+ elif "llama-2" in model_name:
201
+ template_name = "llama_2"
202
+ else:
203
+ template_name = "vicuna_v1"
204
+ new_state = conv_templates[template_name].copy()
205
+ new_state.append_message(new_state.roles[0], state.messages[-2][1])
206
+ new_state.append_message(new_state.roles[1], None)
207
+ state = new_state
208
+
209
+ # Query worker address
210
+ controller_url = args.controller_url
211
+ ret = requests.post(
212
+ controller_url + "/get_worker_address", json={"model": model_name}
213
+ )
214
+ worker_addr = ret.json()["address"]
215
+ logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
216
+
217
+ # No available worker
218
+ if worker_addr == "":
219
+ state.messages[-1][-1] = server_error_msg
220
+ yield (
221
+ state,
222
+ state.to_gradio_chatbot(),
223
+ disable_btn,
224
+ disable_btn,
225
+ disable_btn,
226
+ enable_btn,
227
+ enable_btn,
228
+ )
229
+ return
230
+
231
+ # Construct prompt
232
+ prompt = state.get_prompt()
233
+
234
+ all_images = state.get_images(return_pil=True)
235
+ all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]
236
+ for image, hash in zip(all_images, all_image_hash):
237
+ t = datetime.datetime.now()
238
+ filename = os.path.join(
239
+ LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}", f"{hash}.jpg"
240
+ )
241
+ if not os.path.isfile(filename):
242
+ os.makedirs(os.path.dirname(filename), exist_ok=True)
243
+ image.save(filename)
244
+
245
+ # Make requests
246
+ pload = {
247
+ "model": model_name,
248
+ "prompt": prompt,
249
+ "temperature": float(temperature),
250
+ "top_p": float(top_p),
251
+ "max_new_tokens": min(int(max_new_tokens), 1536),
252
+ "stop": state.sep
253
+ if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT]
254
+ else state.sep2,
255
+ "images": f"List of {len(state.get_images())} images: {all_image_hash}",
256
+ }
257
+ logger.info(f"==== request ====\n{pload}")
258
+
259
+ pload["images"] = state.get_images()
260
+
261
+ state.messages[-1][-1] = "▌"
262
+ yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
263
+
264
+ try:
265
+ # Stream output
266
+ response = requests.post(
267
+ worker_addr + "/worker_generate_stream",
268
+ headers=headers,
269
+ json=pload,
270
+ stream=True,
271
+ timeout=10,
272
+ )
273
+ for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
274
+ if chunk:
275
+ data = json.loads(chunk.decode())
276
+ if data["error_code"] == 0:
277
+ output = data["text"][len(prompt) :].strip()
278
+ state.messages[-1][-1] = output + "▌"
279
+ yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
280
+ else:
281
+ output = data["text"] + f" (error_code: {data['error_code']})"
282
+ state.messages[-1][-1] = output
283
+ yield (state, state.to_gradio_chatbot()) + (
284
+ disable_btn,
285
+ disable_btn,
286
+ disable_btn,
287
+ enable_btn,
288
+ enable_btn,
289
+ )
290
+ return
291
+ time.sleep(0.03)
292
+ except requests.exceptions.RequestException as e:
293
+ state.messages[-1][-1] = server_error_msg
294
+ yield (state, state.to_gradio_chatbot()) + (
295
+ disable_btn,
296
+ disable_btn,
297
+ disable_btn,
298
+ enable_btn,
299
+ enable_btn,
300
+ )
301
+ return
302
+
303
+ state.messages[-1][-1] = state.messages[-1][-1][:-1]
304
+ yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
305
+
306
+ finish_tstamp = time.time()
307
+ logger.info(f"{output}")
308
+
309
+ with open(get_conv_log_filename(), "a") as fout:
310
+ data = {
311
+ "tstamp": round(finish_tstamp, 4),
312
+ "type": "chat",
313
+ "model": model_name,
314
+ "start": round(start_tstamp, 4),
315
+ "finish": round(start_tstamp, 4),
316
+ "state": state.dict(),
317
+ "images": all_image_hash,
318
+ "ip": request.client.host,
319
+ }
320
+ fout.write(json.dumps(data) + "\n")
321
+
322
+
323
+ title_markdown = """
324
+ # 🌋 LLaVA: Large Language and Vision Assistant
325
+ [[Project Page]](https://llava-vl.github.io) [[Paper]](https://arxiv.org/abs/2304.08485) [[Code]](https://github.com/haotian-liu/LLaVA) [[Model]](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)
326
+ """
327
+
328
+ tos_markdown = """
329
+ ### Terms of use
330
+ By using this service, users are required to agree to the following terms:
331
+ The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
332
+ Please click the "Flag" button if you get any inappropriate answer! We will collect those to keep improving our moderator.
333
+ For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
334
+ """
335
+
336
+
337
+ learn_more_markdown = """
338
+ ### License
339
+ The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
340
+ """
341
+
342
+ block_css = """
343
+
344
+ #buttons button {
345
+ min-width: min(120px,100%);
346
+ }
347
+
348
+ """
349
+
350
+
351
+ def build_demo(embed_mode):
352
+ models = get_model_list()
353
+
354
+ textbox = gr.Textbox(
355
+ show_label=False, placeholder="Enter text and press ENTER", container=False
356
+ )
357
+ with gr.Blocks(title="LLaVA", theme=gr.themes.Default(), css=block_css) as demo:
358
+ state = gr.State(default_conversation.copy())
359
+
360
+ if not embed_mode:
361
+ gr.Markdown(title_markdown)
362
+
363
+ with gr.Row():
364
+ with gr.Column(scale=3):
365
+ with gr.Row(elem_id="model_selector_row"):
366
+ model_selector = gr.Dropdown(
367
+ choices=models,
368
+ value=models[0] if len(models) > 0 else "",
369
+ interactive=True,
370
+ show_label=False,
371
+ container=False,
372
+ )
373
+
374
+ imagebox = gr.Image(type="pil")
375
+ image_process_mode = gr.Radio(
376
+ ["Crop", "Resize", "Pad", "Default"],
377
+ value="Default",
378
+ label="Preprocess for non-square image",
379
+ visible=False,
380
+ )
381
+
382
+ cur_dir = os.path.dirname(os.path.abspath(__file__))
383
+ gr.Examples(
384
+ examples=[
385
+ [
386
+ f"{cur_dir}/examples/extreme_ironing.jpg",
387
+ "What is unusual about this image?",
388
+ ],
389
+ [
390
+ f"{cur_dir}/examples/waterview.jpg",
391
+ "What are the things I should be cautious about when I visit here?",
392
+ ],
393
+ ],
394
+ inputs=[imagebox, textbox],
395
+ )
396
+
397
+ with gr.Accordion("Parameters", open=False) as parameter_row:
398
+ temperature = gr.Slider(
399
+ minimum=0.0,
400
+ maximum=1.0,
401
+ value=0.2,
402
+ step=0.1,
403
+ interactive=True,
404
+ label="Temperature",
405
+ )
406
+ top_p = gr.Slider(
407
+ minimum=0.0,
408
+ maximum=1.0,
409
+ value=0.7,
410
+ step=0.1,
411
+ interactive=True,
412
+ label="Top P",
413
+ )
414
+ max_output_tokens = gr.Slider(
415
+ minimum=0,
416
+ maximum=1024,
417
+ value=512,
418
+ step=64,
419
+ interactive=True,
420
+ label="Max output tokens",
421
+ )
422
+
423
+ with gr.Column(scale=8):
424
+ chatbot = gr.Chatbot(
425
+ elem_id="chatbot", label="LLaVA Chatbot", height=550
426
+ )
427
+ with gr.Row():
428
+ with gr.Column(scale=8):
429
+ textbox.render()
430
+ with gr.Column(scale=1, min_width=50):
431
+ submit_btn = gr.Button(value="Send", variant="primary")
432
+ with gr.Row(elem_id="buttons") as button_row:
433
+ upvote_btn = gr.Button(value="👍 Upvote", interactive=False)
434
+ downvote_btn = gr.Button(value="👎 Downvote", interactive=False)
435
+ flag_btn = gr.Button(value="⚠️ Flag", interactive=False)
436
+ # stop_btn = gr.Button(value="⏹️ Stop Generation", interactive=False)
437
+ regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False)
438
+ clear_btn = gr.Button(value="🗑️ Clear history", interactive=False)
439
+
440
+ if not embed_mode:
441
+ gr.Markdown(tos_markdown)
442
+ gr.Markdown(learn_more_markdown)
443
+ url_params = gr.JSON(visible=False)
444
+
445
+ # Register listeners
446
+ btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
447
+ upvote_btn.click(
448
+ upvote_last_response,
449
+ [state, model_selector],
450
+ [textbox, upvote_btn, downvote_btn, flag_btn],
451
+ )
452
+ downvote_btn.click(
453
+ downvote_last_response,
454
+ [state, model_selector],
455
+ [textbox, upvote_btn, downvote_btn, flag_btn],
456
+ )
457
+ flag_btn.click(
458
+ flag_last_response,
459
+ [state, model_selector],
460
+ [textbox, upvote_btn, downvote_btn, flag_btn],
461
+ )
462
+ regenerate_btn.click(
463
+ regenerate,
464
+ [state, image_process_mode],
465
+ [state, chatbot, textbox, imagebox] + btn_list,
466
+ ).then(
467
+ http_bot,
468
+ [state, model_selector, temperature, top_p, max_output_tokens],
469
+ [state, chatbot] + btn_list,
470
+ )
471
+ clear_btn.click(
472
+ clear_history, None, [state, chatbot, textbox, imagebox] + btn_list
473
+ )
474
+
475
+ textbox.submit(
476
+ add_text,
477
+ [state, textbox, imagebox, image_process_mode],
478
+ [state, chatbot, textbox, imagebox] + btn_list,
479
+ ).then(
480
+ http_bot,
481
+ [state, model_selector, temperature, top_p, max_output_tokens],
482
+ [state, chatbot] + btn_list,
483
+ )
484
+ submit_btn.click(
485
+ add_text,
486
+ [state, textbox, imagebox, image_process_mode],
487
+ [state, chatbot, textbox, imagebox] + btn_list,
488
+ ).then(
489
+ http_bot,
490
+ [state, model_selector, temperature, top_p, max_output_tokens],
491
+ [state, chatbot] + btn_list,
492
+ )
493
+
494
+ if args.model_list_mode == "once":
495
+ demo.load(
496
+ load_demo,
497
+ [url_params],
498
+ [state, model_selector],
499
+ _js=get_window_url_params,
500
+ )
501
+ elif args.model_list_mode == "reload":
502
+ demo.load(load_demo_refresh_model_list, None, [state, model_selector])
503
+ else:
504
+ raise ValueError(f"Unknown model list mode: {args.model_list_mode}")
505
+
506
+ return demo
507
+
508
+
509
+ def start_controller():
510
+ logger.info("Starting the controller")
511
+ controller_command = [
512
+ "python",
513
+ "-m",
514
+ "llava.serve.controller",
515
+ "--host",
516
+ "0.0.0.0",
517
+ "--port",
518
+ "10000",
519
+ ]
520
+ return subprocess.Popen(controller_command)
521
+
522
+
523
+ def start_worker(model_path: str):
524
+ logger.info(f"Starting the model worker for the model {model_path}")
525
+ worker_command = [
526
+ "python",
527
+ "-m",
528
+ "llava.serve.model_worker",
529
+ "--host",
530
+ "0.0.0.0",
531
+ "--controller",
532
+ "http://localhost:10000",
533
+ "--model-path",
534
+ model_path,
535
+ ]
536
+ return subprocess.Popen(worker_command)
537
+
538
+
539
+ def preload_models(model_path: str):
540
+ import torch
541
+
542
+ from llava.model import LlavaLlamaForCausalLM
543
+
544
+ model = LlavaLlamaForCausalLM.from_pretrained(
545
+ model_path, low_cpu_mem_usage=True, torch_dtype=torch.float16
546
+ )
547
+ vision_tower = model.get_vision_tower()
548
+ vision_tower.load_model()
549
+
550
+ del vision_tower
551
+ del model
552
+
553
+
554
+ def get_args():
555
+ parser = argparse.ArgumentParser()
556
+ parser.add_argument("--host", type=str, default="0.0.0.0")
557
+ parser.add_argument("--port", type=int)
558
+ parser.add_argument("--controller-url", type=str, default="http://localhost:10000")
559
+ parser.add_argument("--concurrency-count", type=int, default=8)
560
+ parser.add_argument(
561
+ "--model-list-mode", type=str, default="reload", choices=["once", "reload"]
562
+ )
563
+ parser.add_argument("--share", action="store_true")
564
+ parser.add_argument("--moderate", action="store_true")
565
+ parser.add_argument("--embed", action="store_true")
566
+
567
+ args = parser.parse_args()
568
+
569
+ return args
570
+
571
+
572
+ def start_demo(args):
573
+ demo = build_demo(args.embed)
574
+ demo.queue(
575
+ concurrency_count=args.concurrency_count, status_update_rate=10, api_open=False
576
+ ).launch(server_name=args.host, server_port=args.port, share=args.share)
577
+
578
+
579
+ if __name__ == "__main__":
580
+ args = get_args()
581
+ logger.info(f"args: {args}")
582
+
583
+ model_path = "liuhaotian/llava-v1.5-7b"
584
+
585
+ preload_models(model_path)
586
+
587
+ controller_proc = start_controller()
588
+ worker_proc = start_worker(model_path)
589
+
590
+ # Wait for worker and controller to start
591
+ time.sleep(10)
592
+
593
+ try:
594
+ start_demo(args)
595
+ except Exception as e:
596
+ worker_proc.terminate()
597
+ controller_proc.terminate()
598
+
599
+ print(e)
600
+ sys.exit(1)
data/prompts/complex_reasoning/000_caps.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ A man wearing multiple neck ties making a goofy face.
2
+ A man in a white shirt wearing very many ties.
3
+ a man with ties on poses for a picture
4
+ A man wearing multiple ties on his neck.
5
+ A young man smiles while wearing several ties.
6
+
7
+ tie: [0.574, 0.298, 0.752, 0.704]
8
+ tie: [0.464, 0.339, 0.639, 0.789]
9
+ tie: [0.349, 0.363, 0.563, 0.732]
10
+ tie: [0.259, 0.255, 0.668, 0.805]
11
+ person: [0.019, 0.065, 0.962, 0.988]
12
+ person: [0.0, 0.24, 0.214, 1.0]
13
+ tie: [0.316, 0.778, 0.443, 0.867]
14
+ tie: [0.386, 0.707, 0.496, 0.801]
15
+ tie: [0.251, 0.354, 0.402, 0.747]
16
+ tie: [0.44, 0.318, 0.63, 0.397]
17
+ tie: [0.566, 0.33, 0.655, 0.543]
18
+ tie: [0.25, 0.409, 0.359, 0.556]
data/prompts/complex_reasoning/000_conv.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Question:
2
+ What is unusual about this photo?
3
+ ===
4
+ Answer:
5
+ In the photo, the man is wearing a total of ten ties around his neck. This is considered unusual because it is not common for someone to wear multiple ties at once. In a typical setting, a person would wear only one tie. The man's goofy facial expression also adds to the unconventional and humorous nature of the image.
data/prompts/complex_reasoning/001_caps.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ A group of people standing outside of a black vehicle with various luggage.
2
+ Luggage surrounds a vehicle in an underground parking area
3
+ People try to fit all of their luggage in an SUV.
4
+ The sport utility vehicle is parked in the public garage, being packed for a trip
5
+ Some people with luggage near a van that is transporting it.
6
+
7
+ person: [0.681, 0.242, 0.774, 0.694]
8
+ person: [0.63, 0.222, 0.686, 0.516]
9
+ person: [0.444, 0.233, 0.487, 0.34]
10
+ backpack: [0.384, 0.696, 0.485, 0.914]
11
+ backpack: [0.755, 0.413, 0.846, 0.692]
12
+ suitcase: [0.758, 0.413, 0.845, 0.69]
13
+ suitcase: [0.1, 0.497, 0.173, 0.579]
14
+ bicycle: [0.282, 0.363, 0.327, 0.442]
15
+ car: [0.786, 0.25, 0.848, 0.322]
16
+ car: [0.783, 0.27, 0.827, 0.335]
17
+ car: [0.86, 0.254, 0.891, 0.3]
18
+ car: [0.261, 0.101, 0.787, 0.626]
data/prompts/complex_reasoning/001_conv.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Question:
2
+ What challenges do these people face?
3
+ ===
4
+ Answer:
5
+ In the image, a group of people is standing outside a black SUV in a parking area, surrounded by various pieces of luggage, including suitcases and backpacks. They are facing the challenge of fitting all their luggage into the black SUV. There are multiple suitcases and backpacks to be packed, which suggests that the group has a significant amount of belongings to accommodate. They might have to strategize and arrange the luggage efficiently to ensure that everything fits properly into the vehicle. Additionally, they need to consider the comfort of the passengers and visibility while driving, so the placement of the luggage must not obstruct the driver's view or make the passengers uncomfortable during the trip.
data/prompts/complex_reasoning/002_caps.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ There is a movie theater that displays the show times above the doors.
2
+ A red fire hydrant is deep in the snow.
3
+ The fire hydrant is in the snow near a recently plowed sidewalk.
4
+ This city has had a very hard winter with snow.
5
+ A hotel for dogs in the snow in winter.
6
+
7
+ fire hydrant: [0.326, 0.612, 0.426, 0.72]
data/prompts/complex_reasoning/002_conv.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Question:
2
+ What challenges might this city face?
3
+ ===
4
+ Answer:
5
+ The city faces challenges due to the harsh winter conditions and heavy snowfall. In the image, a red fire hydrant is almost buried deep in the snow, which indicates the significant amount of snow the city has experienced. This can lead to various challenges such as difficulties in transportation, increased risk of accidents, and disruptions to daily life. For example, the recently plowed sidewalk near the fire hydrant shows that the city has to constantly clear snow from roads and sidewalks to maintain access and safety for pedestrians and vehicles. Moreover, emergency services, like firefighters, might face challenges accessing crucial equipment, such as fire hydrants, during emergencies due to the snow accumulation. This highlights the importance of effective snow management strategies and preparedness in such cities to minimize the impact of harsh winter conditions on residents and essential services.
data/prompts/complex_reasoning/system_message.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ You are an AI visual assistant that can analyze a single image. You receive five sentences, each describing the same image you are observing. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y.
2
+
3
+ The task is to use the provided caption and bounding box information, create a plausible question about the image, and provide the answer in detail.
4
+
5
+ Create complex questions beyond describing the scene.
6
+ To answer such questions, one should require first understanding the visual content, then based on the background knowledge or reasoning, either explain why the things are happening that way, or provide guides and help to user's request. Make the question challenging by not including the visual content details in the question so that the user needs to reason about that first.
7
+
8
+ Instead of directly mentioning the bounding box coordinates, utilize this data to explain the scene using natural language. Include details like object counts, position of the objects, relative position between the objects.
9
+
10
+ When using the information from the caption and coordinates, directly explain the scene, and do not mention that the information source is the caption or the bounding box. Always answer as if you are directly looking at the image.
data/prompts/conversation/000_caps.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ There is a movie theater that displays the show times above the doors.
2
+ A red fire hydrant is deep in the snow.
3
+ The fire hydrant is in the snow near a recently plowed sidewalk.
4
+ This city has had a very hard winter with snow.
5
+ A hotel for dogs in the snow in winter.
data/prompts/conversation/000_conv.txt ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Question:
2
+ What color is the fire hydrant in the image?
3
+ ===
4
+ Answer:
5
+ The fire hydrant in the image is red.
6
+ ===
7
+ Question:
8
+ Is the fire hydrant fully covered by snow or partially visible?
9
+ ===
10
+ Answer:
11
+ The fire hydrant is partially visible, with a significant portion of it covered by the deep snow.
12
+ ===
13
+ Question:
14
+ What is the condition of the sidewalk near the fire hydrant?
15
+ ===
16
+ Answer:
17
+ The sidewalk near the fire hydrant has been recently plowed, so it is relatively clear of snow compared to the surrounding area.
18
+ ===
19
+ Question:
20
+ What does the image suggest about the winter season in the city?
21
+ ===
22
+ Answer:
23
+ The image suggests that the winter season in the city has been quite harsh and challenging. The significant snow accumulation seen in the image indicates that the city has experienced heavy snowfall, which can lead to various difficulties and disruptions for its residents.
24
+
25
+ For instance, the deep snow covering the fire hydrant could potentially hinder its accessibility in case of emergencies, which poses safety concerns. The recently plowed sidewalk implies that the city's maintenance crews have been actively working to keep the walkways clear and safe for pedestrians, but the sheer amount of snowfall might make it difficult to keep up with the continuous snow removal efforts.
26
+
27
+ Furthermore, such extreme winter conditions can impact transportation, leading to traffic issues, delays in public transit, and increased risks of accidents due to slippery roads. It can also cause problems for businesses and schools, as people might face challenges commuting to work or attending classes. Additionally, the heavy snow can put extra strain on infrastructure, such as roofs and power lines, increasing the likelihood of structural damage or power outages.
28
+
29
+ In conclusion, the image of the red fire hydrant deep in the snow and the recently plowed sidewalk suggest that the city has faced a particularly severe winter season, with substantial snowfall that has likely caused various challenges and disruptions for its residents and infrastructure.
data/prompts/conversation/001_caps.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ A man is skiing in the open snow covered hills
2
+ A skier is making his way into the snow.
3
+ A skier on the foothills of a huge mountain range.
4
+ A skier looks at mountains while standing near a trail sign.
5
+ a single person skiing by an area with a lot of bushes
data/prompts/conversation/001_conv.txt ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Question:
2
+ What is the skier doing?
3
+ ===
4
+ Answer:
5
+ The skier is skiing in the snow-covered hills, making his way through the snow and enjoying the beautiful mountain scenery. They are also standing near a trail sign, which indicates their location on the mountain.
6
+ ===
7
+ Question:
8
+ Are there any other people or skiers visible in the image?
9
+ ===
10
+ Answer:
11
+ No, there is only a single person skiing in the image, and they are surrounded by an area with a lot of bushes.
12
+ ===
13
+ Question:
14
+ What kind of skiing is the person doing? Cross-country or downhill?
15
+ ===
16
+ Answer:
17
+ Based on what I can see in the photo, it is not possible to definitively determine whether the skier is doing cross-country or downhill skiing. However, given the presence of a trail sign and the fact that they are skiing in the foothills, it is more likely that the skier is engaged in cross-country skiing. Downhill skiing typically takes place on more steep slopes and involves ski lifts, whereas cross-country skiing often occurs on relatively flatter terrain with marked trails.
18
+ ===
19
+ Question:
20
+ Can you explain cross-country and downhill skiing a bit more?
21
+ ===
22
+ Answer:
23
+ Cross-country skiing and downhill skiing are two distinct types of skiing, each with its own techniques, equipment, and terrain preferences.
24
+
25
+ 1. Cross-Country Skiing:
26
+ Also known as Nordic skiing, cross-country skiing involves traversing relatively flat terrain or gentle slopes using skis and poles. It is a popular recreational and competitive sport that focuses on endurance, balance, and aerobic fitness. Cross-country skiers use their own locomotion to move across snow-covered terrain, rather than relying on ski lifts or other forms of assistance.
27
+ There are two main techniques in cross-country skiing:
28
+ - Classic: This technique involves skiing in parallel tracks, using a forward-and-backward gliding motion while keeping the skis parallel to each other. The skier uses their poles to propel themselves forward.
29
+ - Skate skiing: This technique is similar to ice skating, where the skier pushes off one ski at an angle and glides on the other. It requires a smooth, groomed surface and is typically faster than the classic technique.
30
+ Cross-country ski equipment consists of lightweight skis, boots, bindings, and poles. The boots are more flexible than those used in downhill skiing, allowing for better ankle movement and control.
31
+
32
+ 2. Downhill Skiing:
33
+ Also known as alpine skiing, downhill skiing involves descending slopes at high speeds using skis and poles for balance and control. This sport is more focused on speed, technique, and navigating challenging terrain, including steep slopes, moguls, and even jumps.
34
+ Downhill skiing can be further categorized into several disciplines, such as slalom, giant slalom, super-G, and downhill racing. Each discipline has its own set of rules, courses, and ski equipment.
35
+ Downhill ski equipment includes heavier and stiffer skis, boots, bindings, and poles than those used in cross-country skiing. The boots are more rigid to provide better support and control during high-speed descents and sharp turns.
36
+
37
+ In summary, cross-country skiing is an endurance-based sport that involves traveling across flat or gently sloping terrain, while downhill skiing is focused on speed and technique as skiers navigate steeper slopes and challenging terrain. Both sports require specialized equipment and techniques, but they offer different experiences and challenges to participants.
data/prompts/conversation/system_message.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ You are an AI visual assistant, and you are seeing a single image. What you see are provided with five sentences, describing the same image you are looking at. Answer all questions as you are seeing the image.
2
+
3
+ Design a conversation between you and a person asking about this photo. The answers should be in a tone that a visual AI assistant is seeing the image and answering the question.
4
+ Ask diverse questions and give corresponding answers.
5
+
6
+ Include questions asking about the visual content of the image, including the object types, counting the objects, object actions, object locations, relative positions between objects, etc. Only include questions that have definite answers:
7
+ (1) one can see the content in the image that the question asks about and can answer confidently;
8
+ (2) one can determine confidently from the image that it is not in the image.
9
+ Do not ask any question that cannot be answered confidently.
10
+
11
+ Also include complex questions that are relevant to the content in the image, for example, asking about background knowledge of the objects in the image, asking to discuss about events happening in the image, etc. Again, do not ask about uncertain details.
12
+ Provide detailed answers when answering complex questions. For example, give detailed examples or reasoning steps to make the content more convincing and well-organized. You can include multiple paragraphs if necessary.
data/prompts/detail_description/000_caps.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ A harbor filled with lots of boats next to a building.
2
+ A bicycle parked in front of several boats at a dock.
3
+ A red bicycle in front of a line of docked white yachts
4
+ A bike sits before boats which sit before a long building.
5
+ A bicycle is a convenient means of land transportation when you live on a boat.
6
+
7
+ bicycle: [0.287, 0.641, 0.507, 0.874]
8
+ bicycle: [0.566, 0.667, 0.63, 0.731]
9
+ boat: [0.318, 0.579, 0.575, 0.724]
10
+ boat: [0.704, 0.607, 0.818, 0.727]
11
+ boat: [0.818, 0.601, 0.942, 0.744]
12
+ boat: [0.002, 0.53, 0.243, 0.71]
13
+ boat: [0.541, 0.611, 0.668, 0.731]
14
+ person: [0.778, 0.527, 0.797, 0.57]
15
+ cup: [0.708, 0.733, 0.724, 0.758]
16
+ boat: [0.236, 0.532, 0.404, 0.64]
17
+ boat: [0.81, 0.632, 0.836, 0.676]
18
+ boat: [0.957, 0.526, 1.0, 0.752]
data/prompts/detail_description/000_conv.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ It is a harbor filled with numerous boats of various sizes docked next to a long building. Among the boats, there are a few white yachts lined up, standing out from the rest. There is a red bicycle prominently parked in front of the line of docked boats, serving as a convenient means of land transportation for those living on the boats. Another bicycle can be seen further back in the scene, near the middle of the harbor.
2
+
3
+ A person is visible near the right side of the harbor, possibly enjoying the view or attending to their boat. Additionally, there is a cup placed on a surface near the middle of the scene.
data/prompts/detail_description/001_caps.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ A group of people standing outside of a black vehicle with various luggage.
2
+ Luggage surrounds a vehicle in an underground parking area
3
+ People try to fit all of their luggage in an SUV.
4
+ The sport utility vehicle is parked in the public garage, being packed for a trip
5
+ Some people with luggage near a van that is transporting it.
6
+
7
+ person: [0.681, 0.242, 0.774, 0.694]
8
+ person: [0.63, 0.222, 0.686, 0.516]
9
+ person: [0.444, 0.233, 0.487, 0.34]
10
+ backpack: [0.384, 0.696, 0.485, 0.914]
11
+ backpack: [0.755, 0.413, 0.846, 0.692]
12
+ suitcase: [0.758, 0.413, 0.845, 0.69]
13
+ suitcase: [0.1, 0.497, 0.173, 0.579]
14
+ bicycle: [0.282, 0.363, 0.327, 0.442]
15
+ car: [0.786, 0.25, 0.848, 0.322]
16
+ car: [0.783, 0.27, 0.827, 0.335]
17
+ car: [0.86, 0.254, 0.891, 0.3]
18
+ car: [0.261, 0.101, 0.787, 0.626]
data/prompts/detail_description/001_conv.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ The image is an underground parking area with a black sport utility vehicle (SUV) parked. There are three people in the scene, with one person standing closer to the left side of the vehicle, another person in the middle, and the third person on the right side. They are all working together to pack their luggage into the SUV for a trip.
2
+
3
+ In the parking area, various luggage items are scattered around the vehicle. There are two backpacks, one located near the left rear wheel and the other closer to the right side of the vehicle. Additionally, there are two suitcases, one on the right side of the car and another further away near the center of the parking area. A bicycle can also be seen on the left side of the vehicle.
4
+
5
+ Other cars are parked around the main SUV, with one car positioned behind it and slightly to the left, another behind and slightly to the right, and the third car further behind on the right side.
data/prompts/detail_description/002_caps.txt ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ A man holds a Wii-mote above his head while another looks on.
2
+ A guy and his friend are playing Nintendo Wii.
3
+ A young man is holding a video game remote over his head.
4
+ two men standing in a room while one plays with a wii mote
5
+ Some guys standing and playing a video game.
6
+
7
+ couch: [0.697, 0.759, 0.995, 1.0]
8
+ dining table: [0.426, 0.755, 1.0, 0.987]
9
+ person: [0.082, 0.252, 0.342, 1.0]
10
+ person: [0.399, 0.085, 0.742, 0.982]
11
+ remote: [0.477, 0.135, 0.516, 0.187]
12
+ sink: [0.016, 0.501, 0.063, 0.52]
13
+ potted plant: [0.798, 0.384, 0.888, 0.645]
14
+ refrigerator: [0.305, 0.389, 0.414, 0.547]
15
+ chair: [0.72, 0.509, 0.858, 0.725]
data/prompts/detail_description/002_conv.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ The image shows two men standing in a room, engaged in playing a video game on a Nintendo Wii console. One of the men is holding a Wii remote above his head with enthusiasm, while the other man looks on, likely enjoying the friendly competition.
2
+
3
+ The room appears to be a living space with a couch located in the background and a dining table nearby. A potted plant can be seen placed close to the couch, and a chair is situated in the middle of the room. The room also features a kitchen area with a sink and a refrigerator visible in the background.
data/prompts/detail_description/system_message.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ You are an AI visual assistant that can analyze a single image. You receive five sentences, each describing the same image you are observing. In addition, specific object locations within the image are given, along with detailed coordinates. These coordinates are in the form of bounding boxes, represented as (x1, y1, x2, y2) with floating numbers ranging from 0 to 1. These values correspond to the top left x, top left y, bottom right x, and bottom right y.
2
+
3
+ Using the provided caption and bounding box information, describe the scene in a detailed manner.
4
+
5
+ Instead of directly mentioning the bounding box coordinates, utilize this data to explain the scene using natural language. Include details like object counts, position of the objects, relative position between the objects.
6
+
7
+ When using the information from the caption and coordinates, directly explain the scene, and do not mention that the information source is the caption or the bounding box. Always answer as if you are directly looking at the image.
docs/Customize_Component.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Customize Components in LLaVA
2
+
3
+ This is an initial guide on how to replace the LLMs, visual encoders, etc. with your choice of components.
4
+
5
+ ## LLM
6
+
7
+ It is quite simple to swap out LLaMA to any other LLMs. You can refer to our implementation of [`llava_llama.py`](https://raw.githubusercontent.com/haotian-liu/LLaVA/main/llava/model/language_model/llava_llama.py) for an example of how to replace the LLM.
8
+
9
+ Although it may seem that it still needs ~100 lines of code, most of them are copied from the original `llama.py` from HF. The only part that is different is to insert some lines for processing the multimodal inputs.
10
+
11
+ In `forward` function, you can see that we call `self.prepare_inputs_labels_for_multimodal` to process the multimodal inputs. This function is defined in `LlavaMetaForCausalLM` and you just need to insert it into the `forward` function of your LLM.
12
+
13
+ In `prepare_inputs_for_generation` function, you can see that we add `images` to the `model_inputs`. This is because we need to pass the images to the LLM during generation.
14
+
15
+ These are basically all the changes you need to make to replace the LLM.
16
+
17
+ ## Visual Encoder
18
+
19
+ You can check out [`clip_encoder.py`](https://github.com/haotian-liu/LLaVA/blob/main/llava/model/multimodal_encoder/clip_encoder.py) on how we implement the CLIP visual encoder.
20
+
docs/Data.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## Data
2
+
3
+ | Data file name | Size |
4
+ | --- | ---: |
5
+ | [llava_instruct_150k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_150k.json) | 229 MB |
6
+ | [llava_instruct_80k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_80k.json) | 229 MB |
7
+ | [conversation_58k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/conversation_58k.json) | 126 MB |
8
+ | [detail_23k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/detail_23k.json) | 20.5 MB |
9
+ | [complex_reasoning_77k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/complex_reasoning_77k.json) | 79.6 MB |
10
+
11
+ ### Pretraining Dataset
12
+ The pretraining dataset used in this release is a subset of CC-3M dataset, filtered with a more balanced concept coverage distribution. Please see [here](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K) for a detailed description of the dataset structure and how to download the images.
13
+
14
+ If you already have CC-3M dataset on your disk, the image names follow this format: `GCC_train_000000000.jpg`. You may edit the `image` field correspondingly if necessary.
15
+
16
+ | Data | Chat File | Meta Data | Size |
17
+ | --- | --- | --- | ---: |
18
+ | CC-3M Concept-balanced 595K | [chat.json](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K/blob/main/chat.json) | [metadata.json](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K/blob/main/metadata.json) | 211 MB
19
+ | LAION/CC/SBU BLIP-Caption Concept-balanced 558K | [blip_laion_cc_sbu_558k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Pretrain/blob/main/blip_laion_cc_sbu_558k.json) | [metadata.json](#) | 181 MB
20
+
21
+ **Important notice**: Upon the request from the community, as ~15% images of the original CC-3M dataset are no longer accessible, we upload [`images.zip`](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K/blob/main/images.zip) for better reproducing our work in research community. It must not be used for any other purposes. The use of these images must comply with the CC-3M license. This may be taken down at any time when requested by the original CC-3M dataset owner or owners of the referenced images.
22
+
23
+ ### GPT-4 Prompts
24
+
25
+ We provide our prompts and few-shot samples for GPT-4 queries, to better facilitate research in this domain. Please check out the [`prompts`](playground/data/prompts) folder for three kinds of questions: conversation, detail description, and complex reasoning.
26
+
27
+ They are organized in a format of `system_message.txt` for system message, pairs of `abc_caps.txt` for few-shot sample user input, and `abc_conv.txt` for few-shot sample reference output.
28
+
29
+ Note that you may find them in different format. For example, `conversation` is in `jsonl`, and detail description is answer-only. The selected format in our preliminary experiments works slightly better than a limited set of alternatives that we tried: `jsonl`, more natural format, answer-only. If interested, you may try other variants or conduct more careful study in this. Contributions are welcomed!
docs/LLaVA_Bench.md ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LLaVA-Bench [[Download](https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild)]
2
+
3
+ **-Introduction-** Large commercial multimodal chatbots have been released in this week, including
4
+ - [Multimodal Bing-Chat by Microsoft](https://blogs.bing.com/search/july-2023/Bing-Chat-Enterprise-announced,-multimodal-Visual-Search-rolling-out-to-Bing-Chat) (July 18, 2023)
5
+ - [Multimodal Bard by Google](https://bard.google.com/).
6
+
7
+ These chatbots are presumably supported by proprietary large multimodal models (LMM). Compared with the open-source LMM such as LLaVA, proprietary LMM represent the scaling success upperbound of the current SoTA techniques. They share the goal of developing multimodal chatbots that follow human intents to complete various daily-life visual tasks in the wild. While it remains less unexplored how to evaluate multimodal chat ability, it provides useful feedback to study open-source LMMs against the commercial multimodal chatbots. In addition to the *LLaVA-Bench (COCO)* dataset we used to develop the early versions of LLaVA, we are releasing [*LLaVA-Bench (In-the-Wild)*](https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild) to the community for the public use.
8
+
9
+ ## LLaVA-Bench (In-the-Wild *[Ongoing work]*)
10
+
11
+ To evaluate the model's capability in more challenging tasks and generalizability to novel domains, we collect a diverse set of 24 images with 60 questions in total, including indoor and outdoor scenes, memes, paintings, sketches, etc, and associate each image with a highly-detailed and manually-curated description and a proper selection of questions. Such design also assesses the model's robustness to different prompts. In this release, we also categorize questions into three categories: conversation (simple QA), detailed description, and complex reasoning. We continue to expand and improve the diversity of the LLaVA-Bench (In-the-Wild). We manually query Bing-Chat and Bard to get the responses.
12
+
13
+ ### Results
14
+
15
+ The score is measured by comparing against a reference answer generated by text-only GPT-4. It is generated by feeding the question, along with the ground truth image annotations as the context. A text-only GPT-4 evaluator rates both answers. We query GPT-4 by putting the reference answer first, and then the answer generated by the candidate model. We upload images at their original resolution to Bard and Bing-Chat to obtain the results.
16
+
17
+ | Approach | Conversation | Detail | Reasoning | Overall |
18
+ |----------------|--------------|--------|-----------|---------|
19
+ | Bard-0718 | 83.7 | 69.7 | 78.7 | 77.8 |
20
+ | Bing-Chat-0629 | 59.6 | 52.2 | 90.1 | 71.5 |
21
+ | LLaVA-13B-v1-336px-0719 (beam=1) | 64.3 | 55.9 | 81.7 | 70.1 |
22
+ | LLaVA-13B-v1-336px-0719 (beam=5) | 68.4 | 59.9 | 84.3 | 73.5 |
23
+
24
+ Note that Bard sometimes refuses to answer questions about images containing humans, and Bing-Chat blurs the human faces in the images. We also provide the benchmark score for the subset without humans.
25
+
26
+ | Approach | Conversation | Detail | Reasoning | Overall |
27
+ |----------------|--------------|--------|-----------|---------|
28
+ | Bard-0718 | 94.9 | 74.3 | 84.3 | 84.6 |
29
+ | Bing-Chat-0629 | 55.8 | 53.6 | 93.5 | 72.6 |
30
+ | LLaVA-13B-v1-336px-0719 (beam=1) | 62.2 | 56.4 | 82.2 | 70.0 |
31
+ | LLaVA-13B-v1-336px-0719 (beam=5) | 65.6 | 61.7 | 85.0 | 73.6 |
docs/LLaVA_from_LLaMA2.md ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LLaVA (based on Llama 2 LLM, Preview)
2
+
3
+ *NOTE: This is a technical preview. We are still running hyperparameter search, and will release the final model soon. If you'd like to contribute to this, please contact us.*
4
+
5
+ :llama: **-Introduction-** [Llama 2 is an open-source LLM released by Meta AI](https://about.fb.com/news/2023/07/llama-2/) today (July 18, 2023). Compared with its early version [Llama 1](https://ai.meta.com/blog/large-language-model-llama-meta-ai/), Llama 2 is more favored in ***stronger language performance***, ***longer context window***, and importantly ***commercially usable***! While Llama 2 is changing the LLM market landscape in the language space, its multimodal ability remains unknown. We quickly develop the LLaVA variant based on the latest Llama 2 checkpoints, and release it to the community for the public use.
6
+
7
+ You need to apply for and download the lastest Llama 2 checkpoints to start your own training (apply [here](https://ai.meta.com/resources/models-and-libraries/llama-downloads/))
8
+
9
+
10
+ ## Training
11
+
12
+ Please checkout [`pretrain.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/pretrain.sh), [`finetune.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune.sh), [`finetune_lora.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_lora.sh).
13
+
14
+ ## LLaVA (based on Llama 2), What is different?
15
+
16
+ :volcano: How is the new LLaVA based on Llama 2 different from Llama 1? The comparisons of the training process are described:
17
+ - **Pre-training**. The pre-trained base LLM is changed from Llama 1 to Llama 2
18
+ - **Language instruction-tuning**. The previous LLaVA model starts with Vicuna, which is instruct tuned on ShareGPT data from Llama 1; The new LLaVA model starts with Llama 2 Chat, which is an instruct tuned checkpoint on dialogue data from Llama 2.
19
+ - **Multimodal instruction-tuning**. The same LLaVA-Lighting process is applied.
20
+
21
+
22
+ ### Results
23
+
24
+ - Llama 2 is better at following the instructions of role playing; Llama 2 fails in following the instructions of translation
25
+ - The quantitative evaluation on [LLaVA-Bench](https://github.com/haotian-liu/LLaVA/blob/main/docs/LLaVA_Bench.md) demonstrates on-par performance between Llama 2 and Llama 1 in LLaVA's multimodal chat ability.
26
+
27
+
28
+ <img src="../images/llava_example_cmp.png" width="100%">
29
+
docs/LoRA.md ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # LLaVA (LoRA, Preview)
2
+
3
+ NOTE: This is a technical preview, and is not yet ready for production use. We are still running hyperparameter search for the LoRA model, and will release the final model soon. If you'd like to contribute to this, please contact us.
4
+
5
+ You need latest code base for LoRA support (instructions [here](https://github.com/haotian-liu/LLaVA#upgrade-to-latest-code-base))
6
+
7
+ ## Demo (Web UI)
8
+
9
+ Please execute each of the command below one by one (after the previous one has finished). The commands are the same as launching other demos except for an additional `--model-base` flag to specify the base model to use. Please make sure the base model corresponds to the LoRA checkpoint that you are using. For this technical preview, you need Vicuna v1.1 (7B) checkpoint (if you do not have that already, follow the instructions [here](https://github.com/lm-sys/FastChat#vicuna-weights)).
10
+
11
+ #### Launch a controller
12
+ ```Shell
13
+ python -m llava.serve.controller --host 0.0.0.0 --port 10000
14
+ ```
15
+
16
+ #### Launch a gradio web server.
17
+ ```Shell
18
+ python -m llava.serve.gradio_web_server --controller http://localhost:10000 --model-list-mode reload
19
+ ```
20
+ You just launched the Gradio web interface. Now, you can open the web interface with the URL printed on the screen. You may notice that there is no model in the model list. Do not worry, as we have not launched any model worker yet. It will be automatically updated when you launch a model worker.
21
+
22
+ #### Launch a model worker
23
+ ```Shell
24
+ python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/llava-vicuna-7b-v1.1-lcs_558k-instruct_80k_3e-lora-preview-alpha --model-base /path/to/vicuna-v1.1
25
+ ```
26
+ Wait until the process finishes loading the model and you see "Uvicorn running on ...". Now, refresh your Gradio web UI, and you will see the model you just launched in the model list.
27
+
28
+ You can launch as many workers as you want, and compare between different model checkpoints in the same Gradio interface. Please keep the `--controller` the same, and modify the `--port` and `--worker` to a different port number for each worker.
29
+
30
+
31
+ ## Training
32
+
33
+ Please see sample training scripts for [LoRA](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_lora.sh) and [QLoRA](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_qlora.sh).
34
+
35
+ We provide sample DeepSpeed configs, [`zero3.json`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/zero3.json) is more like PyTorch FSDP, and [`zero3_offload.json`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/zero3_offload.json) can further save memory consumption by offloading parameters to CPU. `zero3.json` is usually faster than `zero3_offload.json` but requires more GPU memory, therefore, we recommend trying `zero3.json` first, and if you run out of GPU memory, try `zero3_offload.json`. You can also tweak the `per_device_train_batch_size` and `gradient_accumulation_steps` in the config to save memory, and just to make sure that `per_device_train_batch_size` and `gradient_accumulation_steps` remains the same.
36
+
37
+ If you are having issues with ZeRO-3 configs, and there are enough VRAM, you may try [`zero2.json`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/zero2.json). This consumes slightly more memory than ZeRO-3, and behaves more similar to PyTorch FSDP, while still supporting parameter-efficient tuning.
38
+
39
+ ## Create Merged Checkpoints
40
+
41
+ ```Shell
42
+ python scripts/merge_lora_weights.py \
43
+ --model-path /path/to/lora_model \
44
+ --model-base /path/to/base_model \
45
+ --save-model-path /path/to/merge_model
46
+ ```
docs/MODEL_ZOO.md ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Model Zoo
2
+
3
+ **To Use LLaVA-1.5 checkpoints, your llava package version must be newer than 1.1.0. [Instructions](https://github.com/haotian-liu/LLaVA#upgrade-to-latest-code-base) on how to upgrade.**
4
+
5
+ If you are interested in including any other details in Model Zoo, please open an issue :)
6
+
7
+ The model weights below are *merged* weights. You do not need to apply delta. The usage of LLaVA checkpoints should comply with the base LLM's model license: [LLaMA](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md).
8
+
9
+ ## LLaVA-v1.5
10
+
11
+ | Version | Size | Schedule | Checkpoint | VQAv2 | GQA | VizWiz | SQA | T-VQA | POPE | MME | MM-Bench | MM-Bench-CN | SEED | LLaVA-Bench-Wild | MM-Vet |
12
+ |----------|----------|-----------|-----------|---|---|---|---|---|---|---|---|---|---|---|---|
13
+ | LLaVA-1.5 | 7B | full_ft-1e | [liuhaotian/llava-v1.5-7b](https://huggingface.co/liuhaotian/llava-v1.5-7b) | 78.5 | 62.0 | 50.0 | 66.8 | 58.2 | 85.9 | 1510.7 | 64.3 | 58.3 | 58.6 | 63.4 | 30.5 |
14
+ | LLaVA-1.5 | 13B | full_ft-1e | [liuhaotian/llava-v1.5-13b](https://huggingface.co/liuhaotian/llava-v1.5-13b) | 80.0 | 63.3 | 53.6 | 71.6 | 61.3 | 85.9 | 1531.3 | 67.7 | 63.6 | 61.6 | 70.7 | 35.4 |
15
+ | LLaVA-1.5 | 7B | lora-1e | coming soon |
16
+ | LLaVA-1.5 | 13B | lora-1e | coming soon |
17
+
18
+ <p align="center">
19
+ <img src="../images/llava_v1_5_radar.jpg" width="500px"> <br>
20
+ LLaVA-1.5 achieves SoTA performance across 11 benchmarks.
21
+ </p>
22
+
23
+
24
+ ## LLaVA-v1
25
+
26
+ *Note: We recommend using the most capable LLaVA-v1.5 series above for the best performance.*
27
+
28
+ | Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | LLaVA-Bench-Conv | LLaVA-Bench-Detail | LLaVA-Bench-Complex | LLaVA-Bench-Overall | Download |
29
+ |----------|----------------|---------------|----------------------|-----------------|--------------------|------------------|--------------------|---------------------|---------------------|---------------------|
30
+ | Vicuna-13B-v1.3 | CLIP-L-336px | LCS-558K | 1e | LLaVA-Instruct-80K | proj-1e, lora-1e | 64.3 | 55.9 | 81.7 | 70.1 | [LoRA](https://huggingface.co/liuhaotian/llava-v1-0719-336px-lora-vicuna-13b-v1.3) [LoRA-Merged](https://huggingface.co/liuhaotian/llava-v1-0719-336px-lora-merge-vicuna-13b-v1.3) |
31
+ | LLaMA-2-13B-Chat | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | full_ft-1e | 56.7 | 58.6 | 80.0 | 67.9 | [ckpt](https://huggingface.co/liuhaotian/llava-llama-2-13b-chat-lightning-preview) |
32
+ | LLaMA-2-7B-Chat | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | lora-1e | 51.2 | 58.9 | 71.6 | 62.8 | [LoRA](https://huggingface.co/liuhaotian/llava-llama-2-7b-chat-lightning-lora-preview) |
33
+
34
+
35
+ ## Projector weights
36
+
37
+ The model weights below are projector weights we have pretrained. You can use these projector weights for visual instruction tuning. We'll add more projector weights into model zoo very soon.
38
+
39
+ **NOTE**: These projector weights are only compatible with the `llava>=1.0.0`, please check out the latest code base if your local code version is below `v1.0.0`.
40
+
41
+ **NOTE**: When you use our pretrained projector for visual instruction tuning, it is very important to **use the same base LLM and vision encoder** as the one we used for pretraining the projector. Otherwise, the performance will be very bad.
42
+
43
+ When using these projector weights to instruction tune your LMM, please make sure that these options are correctly set as follows,
44
+
45
+ ```Shell
46
+ --mm_use_im_start_end False
47
+ --mm_use_im_patch_token False
48
+ ```
49
+
50
+ | Base LLM | Vision Encoder | Projection | Pretrain Data | Pretraining schedule | Download |
51
+ |----------|----------------|---------------|----------------------|----------|----------|
52
+ | Vicuna-13B-v1.5 | CLIP-L-336px | MLP-2x | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-v1.5-mlp2x-336px-pretrain-vicuna-13b-v1.5) |
53
+ | Vicuna-7B-v1.5 | CLIP-L-336px | MLP-2x | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-v1.5-mlp2x-336px-pretrain-vicuna-7b-v1.5) |
54
+ | LLaMA-2-13B-Chat | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-llama-2-13b-chat) |
55
+ | LLaMA-2-7B-Chat | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-llama-2-7b-chat) |
56
+ | LLaMA-2-13B-Chat | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-llama-2-13b-chat) |
57
+ | LLaMA-2-7B-Chat | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-llama-2-7b-chat) |
58
+ | Vicuna-13B-v1.3 | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-vicuna-13b-v1.3) |
59
+ | Vicuna-7B-v1.3 | CLIP-L-336px | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-336px-pretrain-vicuna-7b-v1.3) |
60
+ | Vicuna-13B-v1.3 | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-vicuna-13b-v1.3) |
61
+ | Vicuna-7B-v1.3 | CLIP-L | Linear | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/llava-pretrain-vicuna-7b-v1.3) |
62
+
63
+
64
+ ## Science QA Checkpoints
65
+
66
+ | Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | Download |
67
+ |----------|----------------|---------------|----------------------|-----------------|--------------------|---------------------|
68
+ | Vicuna-13B-v1.3 | CLIP-L | LCS-558K | 1e | ScienceQA | full_ft-12e | [ckpt](https://huggingface.co/liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3) |
69
+
70
+
71
+ ## Legacy Models (merged weights)
72
+
73
+ The model weights below are *merged* weights. You do not need to apply delta. The usage of LLaVA checkpoints should comply with the base LLM's model license.
74
+
75
+ | Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | Download |
76
+ |----------|----------------|---------------|----------------------|-----------------|--------------------|------------------|
77
+ | MPT-7B-Chat | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | full_ft-1e | [preview](https://huggingface.co/liuhaotian/LLaVA-Lightning-MPT-7B-preview) |
78
+
79
+
80
+ ## Legacy Models (delta weights)
81
+
82
+ The model weights below are *delta* weights. The usage of LLaVA checkpoints should comply with the base LLM's model license: [LLaMA](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md).
83
+
84
+ You can add our delta to the original LLaMA weights to obtain the LLaVA weights.
85
+
86
+ Instructions:
87
+
88
+ 1. Get the original LLaMA weights in the huggingface format by following the instructions [here](https://huggingface.co/docs/transformers/main/model_doc/llama).
89
+ 2. Use the following scripts to get LLaVA weights by applying our delta. It will automatically download delta weights from our Hugging Face account. In the script below, we use the delta weights of [`liuhaotian/LLaVA-7b-delta-v0`](https://huggingface.co/liuhaotian/LLaVA-7b-delta-v0) as an example. It can be adapted for other delta weights by changing the `--delta` argument (and base/target accordingly).
90
+
91
+ ```bash
92
+ python3 -m llava.model.apply_delta \
93
+ --base /path/to/llama-7b \
94
+ --target /output/path/to/LLaVA-7B-v0 \
95
+ --delta liuhaotian/LLaVA-7b-delta-v0
96
+ ```
97
+
98
+ | Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Finetuning Data | Finetuning schedule | Download |
99
+ |----------|----------------|---------------|----------------------|-----------------|--------------------|------------------|
100
+ | Vicuna-13B-v1.1 | CLIP-L | CC-595K | 1e | LLaVA-Instruct-158K | full_ft-3e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-13b-delta-v1-1) |
101
+ | Vicuna-7B-v1.1 | CLIP-L | LCS-558K | 1e | LLaVA-Instruct-80K | full_ft-1e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-Lightning-7B-delta-v1-1) |
102
+ | Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | LLaVA-Instruct-158K | full_ft-3e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-13b-delta-v0) |
103
+ | Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | ScienceQA | full_ft-12e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-13b-delta-v0-science_qa) |
104
+ | Vicuna-7B-v0 | CLIP-L | CC-595K | 1e | LLaVA-Instruct-158K | full_ft-3e | [delta-weights](https://huggingface.co/liuhaotian/LLaVA-7b-delta-v0) |
105
+
106
+
107
+
108
+ ## Legacy Projector weights
109
+
110
+ The following projector weights are deprecated, and the support for them may be removed in the future. They do not support zero-shot inference. Please use the projector weights in the [table above](#projector-weights) if possible.
111
+
112
+ **NOTE**: When you use our pretrained projector for visual instruction tuning, it is very important to **use the same base LLM and vision encoder** as the one we used for pretraining the projector. Otherwise, the performance will be very bad.
113
+
114
+ When using these projector weights to instruction tune your LMM, please make sure that these options are correctly set as follows,
115
+
116
+ ```Shell
117
+ --mm_use_im_start_end True
118
+ --mm_use_im_patch_token False
119
+ ```
120
+
121
+ | Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Download |
122
+ |----------|----------------|---------------|----------------------|----------|
123
+ | Vicuna-7B-v1.1 | CLIP-L | LCS-558K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-7b-pretrain-projector-v1-1-LCS-558K-blip_caption.bin) |
124
+ | Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-13b-pretrain-projector-v0-CC3M-595K-original_caption.bin) |
125
+ | Vicuna-7B-v0 | CLIP-L | CC-595K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-7b-pretrain-projector-v0-CC3M-595K-original_caption.bin) |
126
+
127
+ When using these projector weights to instruction tune your LMM, please make sure that these options are correctly set as follows,
128
+
129
+ ```Shell
130
+ --mm_use_im_start_end False
131
+ --mm_use_im_patch_token False
132
+ ```
133
+
134
+ | Base LLM | Vision Encoder | Pretrain Data | Pretraining schedule | Download |
135
+ |----------|----------------|---------------|----------------------|----------|
136
+ | Vicuna-13B-v0 | CLIP-L | CC-595K | 1e | [projector](https://huggingface.co/liuhaotian/LLaVA-Pretrained-Projectors/blob/main/LLaVA-13b-pretrain-projector-v0-CC3M-595K-original_caption-no_im_token.bin) |
docs/ScienceQA.md ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ### ScienceQA
2
+
3
+ #### Prepare Data
4
+ 1. Please see ScienceQA [repo](https://github.com/lupantech/ScienceQA) for setting up the dataset.
5
+ 2. Generate ScienceQA dataset for LLaVA conversation-style format.
6
+
7
+ ```Shell
8
+ python scripts/convert_sqa_to_llava.py \
9
+ convert_to_llava \
10
+ --base-dir /path/to/ScienceQA/data/scienceqa \
11
+ --prompt-format "QCM-LEA" \
12
+ --split {train,val,minival,test,minitest}
13
+ ```
14
+
15
+ #### Training
16
+
17
+ 1. Pretraining
18
+
19
+ You can download our pretrained projector weights from our [Model Zoo](), or train your own projector weights using [`pretrain.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/pretrain.sh).
20
+
21
+ 2. Finetuning
22
+
23
+ See [`finetune_sqa.sh`](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_sqa.sh).
24
+
25
+ #### Evaluation
26
+
27
+ 1. Multiple-GPU inference
28
+ You may evaluate this with multiple GPUs, and concatenate the generated jsonl files. Please refer to our script for [batch evaluation](https://github.com/haotian-liu/LLaVA/blob/main/scripts/sqa_eval_batch.sh) and [results gathering](https://github.com/haotian-liu/LLaVA/blob/main/scripts/sqa_eval_gather.sh).
29
+
30
+ 2. Single-GPU inference
31
+
32
+ (a) Generate LLaVA responses on ScienceQA dataset
33
+
34
+ ```Shell
35
+ python -m llava.eval.model_vqa_science \
36
+ --model-path liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3 \
37
+ --question-file /path/to/ScienceQA/data/scienceqa/llava_test_QCM-LEA.json \
38
+ --image-folder /path/to/ScienceQA/data/scienceqa/images/test \
39
+ --answers-file vqa/results/ScienceQA/test_llava-13b.jsonl \
40
+ --conv-mode llava_v1
41
+ ```
42
+
43
+ (b) Evaluate the generated responses
44
+
45
+ ```Shell
46
+ python eval_science_qa.py \
47
+ --base-dir /path/to/ScienceQA/data/scienceqa \
48
+ --result-file vqa/results/ScienceQA/test_llava-13b.jsonl \
49
+ --output-file vqa/results/ScienceQA/test_llava-13b_output.json \
50
+ --output-result vqa/results/ScienceQA/test_llava-13b_result.json \
51
+ ```
52
+
53
+ For reference, we attach our prediction file [`test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json`](https://github.com/haotian-liu/LLaVA/blob/main/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json) and [`test_sqa_llava_13b_v0.json`](https://github.com/haotian-liu/LLaVA/blob/main/llava/eval/table/results/test_sqa_llava_13b_v0.json) for comparison when reproducing our results, as well as for further analysis in detail.
examples/extreme_ironing.jpg ADDED
examples/waterview.jpg ADDED
llava/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .model import LlavaLlamaForCausalLM
llava/constants.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ CONTROLLER_HEART_BEAT_EXPIRATION = 30
2
+ WORKER_HEART_BEAT_INTERVAL = 15
3
+
4
+ LOGDIR = "."
5
+
6
+ # Model Constants
7
+ IGNORE_INDEX = -100
8
+ IMAGE_TOKEN_INDEX = -200
9
+ DEFAULT_IMAGE_TOKEN = "<image>"
10
+ DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
11
+ DEFAULT_IM_START_TOKEN = "<im_start>"
12
+ DEFAULT_IM_END_TOKEN = "<im_end>"
llava/conversation.py ADDED
@@ -0,0 +1,381 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import dataclasses
2
+ from enum import auto, Enum
3
+ from typing import List, Tuple
4
+
5
+
6
+ class SeparatorStyle(Enum):
7
+ """Different separator style."""
8
+ SINGLE = auto()
9
+ TWO = auto()
10
+ MPT = auto()
11
+ PLAIN = auto()
12
+ LLAMA_2 = auto()
13
+
14
+
15
+ @dataclasses.dataclass
16
+ class Conversation:
17
+ """A class that keeps all conversation history."""
18
+ system: str
19
+ roles: List[str]
20
+ messages: List[List[str]]
21
+ offset: int
22
+ sep_style: SeparatorStyle = SeparatorStyle.SINGLE
23
+ sep: str = "###"
24
+ sep2: str = None
25
+ version: str = "Unknown"
26
+
27
+ skip_next: bool = False
28
+
29
+ def get_prompt(self):
30
+ messages = self.messages
31
+ if len(messages) > 0 and type(messages[0][1]) is tuple:
32
+ messages = self.messages.copy()
33
+ init_role, init_msg = messages[0].copy()
34
+ init_msg = init_msg[0].replace("<image>", "").strip()
35
+ if 'mmtag' in self.version:
36
+ messages[0] = (init_role, init_msg)
37
+ messages.insert(0, (self.roles[0], "<Image><image></Image>"))
38
+ messages.insert(1, (self.roles[1], "Received."))
39
+ else:
40
+ messages[0] = (init_role, "<image>\n" + init_msg)
41
+
42
+ if self.sep_style == SeparatorStyle.SINGLE:
43
+ ret = self.system + self.sep
44
+ for role, message in messages:
45
+ if message:
46
+ if type(message) is tuple:
47
+ message, _, _ = message
48
+ ret += role + ": " + message + self.sep
49
+ else:
50
+ ret += role + ":"
51
+ elif self.sep_style == SeparatorStyle.TWO:
52
+ seps = [self.sep, self.sep2]
53
+ ret = self.system + seps[0]
54
+ for i, (role, message) in enumerate(messages):
55
+ if message:
56
+ if type(message) is tuple:
57
+ message, _, _ = message
58
+ ret += role + ": " + message + seps[i % 2]
59
+ else:
60
+ ret += role + ":"
61
+ elif self.sep_style == SeparatorStyle.MPT:
62
+ ret = self.system + self.sep
63
+ for role, message in messages:
64
+ if message:
65
+ if type(message) is tuple:
66
+ message, _, _ = message
67
+ ret += role + message + self.sep
68
+ else:
69
+ ret += role
70
+ elif self.sep_style == SeparatorStyle.LLAMA_2:
71
+ wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n"
72
+ wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
73
+ ret = ""
74
+
75
+ for i, (role, message) in enumerate(messages):
76
+ if i == 0:
77
+ assert message, "first message should not be none"
78
+ assert role == self.roles[0], "first message should come from user"
79
+ if message:
80
+ if type(message) is tuple:
81
+ message, _, _ = message
82
+ if i == 0: message = wrap_sys(self.system) + message
83
+ if i % 2 == 0:
84
+ message = wrap_inst(message)
85
+ ret += self.sep + message
86
+ else:
87
+ ret += " " + message + " " + self.sep2
88
+ else:
89
+ ret += ""
90
+ ret = ret.lstrip(self.sep)
91
+ elif self.sep_style == SeparatorStyle.PLAIN:
92
+ seps = [self.sep, self.sep2]
93
+ ret = self.system
94
+ for i, (role, message) in enumerate(messages):
95
+ if message:
96
+ if type(message) is tuple:
97
+ message, _, _ = message
98
+ ret += message + seps[i % 2]
99
+ else:
100
+ ret += ""
101
+ else:
102
+ raise ValueError(f"Invalid style: {self.sep_style}")
103
+
104
+ return ret
105
+
106
+ def append_message(self, role, message):
107
+ self.messages.append([role, message])
108
+
109
+ def get_images(self, return_pil=False):
110
+ images = []
111
+ for i, (role, msg) in enumerate(self.messages[self.offset:]):
112
+ if i % 2 == 0:
113
+ if type(msg) is tuple:
114
+ import base64
115
+ from io import BytesIO
116
+ from PIL import Image
117
+ msg, image, image_process_mode = msg
118
+ if image_process_mode == "Pad":
119
+ def expand2square(pil_img, background_color=(122, 116, 104)):
120
+ width, height = pil_img.size
121
+ if width == height:
122
+ return pil_img
123
+ elif width > height:
124
+ result = Image.new(pil_img.mode, (width, width), background_color)
125
+ result.paste(pil_img, (0, (width - height) // 2))
126
+ return result
127
+ else:
128
+ result = Image.new(pil_img.mode, (height, height), background_color)
129
+ result.paste(pil_img, ((height - width) // 2, 0))
130
+ return result
131
+ image = expand2square(image)
132
+ elif image_process_mode in ["Default", "Crop"]:
133
+ pass
134
+ elif image_process_mode == "Resize":
135
+ image = image.resize((336, 336))
136
+ else:
137
+ raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
138
+ max_hw, min_hw = max(image.size), min(image.size)
139
+ aspect_ratio = max_hw / min_hw
140
+ max_len, min_len = 800, 400
141
+ shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
142
+ longest_edge = int(shortest_edge * aspect_ratio)
143
+ W, H = image.size
144
+ if longest_edge != max(image.size):
145
+ if H > W:
146
+ H, W = longest_edge, shortest_edge
147
+ else:
148
+ H, W = shortest_edge, longest_edge
149
+ image = image.resize((W, H))
150
+ if return_pil:
151
+ images.append(image)
152
+ else:
153
+ buffered = BytesIO()
154
+ image.save(buffered, format="PNG")
155
+ img_b64_str = base64.b64encode(buffered.getvalue()).decode()
156
+ images.append(img_b64_str)
157
+ return images
158
+
159
+ def to_gradio_chatbot(self):
160
+ ret = []
161
+ for i, (role, msg) in enumerate(self.messages[self.offset:]):
162
+ if i % 2 == 0:
163
+ if type(msg) is tuple:
164
+ import base64
165
+ from io import BytesIO
166
+ msg, image, image_process_mode = msg
167
+ max_hw, min_hw = max(image.size), min(image.size)
168
+ aspect_ratio = max_hw / min_hw
169
+ max_len, min_len = 800, 400
170
+ shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
171
+ longest_edge = int(shortest_edge * aspect_ratio)
172
+ W, H = image.size
173
+ if H > W:
174
+ H, W = longest_edge, shortest_edge
175
+ else:
176
+ H, W = shortest_edge, longest_edge
177
+ image = image.resize((W, H))
178
+ buffered = BytesIO()
179
+ image.save(buffered, format="JPEG")
180
+ img_b64_str = base64.b64encode(buffered.getvalue()).decode()
181
+ img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
182
+ msg = img_str + msg.replace('<image>', '').strip()
183
+ ret.append([msg, None])
184
+ else:
185
+ ret.append([msg, None])
186
+ else:
187
+ ret[-1][-1] = msg
188
+ return ret
189
+
190
+ def copy(self):
191
+ return Conversation(
192
+ system=self.system,
193
+ roles=self.roles,
194
+ messages=[[x, y] for x, y in self.messages],
195
+ offset=self.offset,
196
+ sep_style=self.sep_style,
197
+ sep=self.sep,
198
+ sep2=self.sep2,
199
+ version=self.version)
200
+
201
+ def dict(self):
202
+ if len(self.get_images()) > 0:
203
+ return {
204
+ "system": self.system,
205
+ "roles": self.roles,
206
+ "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
207
+ "offset": self.offset,
208
+ "sep": self.sep,
209
+ "sep2": self.sep2,
210
+ }
211
+ return {
212
+ "system": self.system,
213
+ "roles": self.roles,
214
+ "messages": self.messages,
215
+ "offset": self.offset,
216
+ "sep": self.sep,
217
+ "sep2": self.sep2,
218
+ }
219
+
220
+
221
+ conv_vicuna_v0 = Conversation(
222
+ system="A chat between a curious human and an artificial intelligence assistant. "
223
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.",
224
+ roles=("Human", "Assistant"),
225
+ messages=(
226
+ ("Human", "What are the key differences between renewable and non-renewable energy sources?"),
227
+ ("Assistant",
228
+ "Renewable energy sources are those that can be replenished naturally in a relatively "
229
+ "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
230
+ "Non-renewable energy sources, on the other hand, are finite and will eventually be "
231
+ "depleted, such as coal, oil, and natural gas. Here are some key differences between "
232
+ "renewable and non-renewable energy sources:\n"
233
+ "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
234
+ "energy sources are finite and will eventually run out.\n"
235
+ "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
236
+ "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
237
+ "and other negative effects.\n"
238
+ "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
239
+ "have lower operational costs than non-renewable sources.\n"
240
+ "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
241
+ "locations than non-renewable sources.\n"
242
+ "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
243
+ "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
244
+ "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
245
+ "non-renewable sources are not, and their depletion can lead to economic and social instability.\n")
246
+ ),
247
+ offset=2,
248
+ sep_style=SeparatorStyle.SINGLE,
249
+ sep="###",
250
+ )
251
+
252
+ conv_vicuna_v1 = Conversation(
253
+ system="A chat between a curious user and an artificial intelligence assistant. "
254
+ "The assistant gives helpful, detailed, and polite answers to the user's questions.",
255
+ roles=("USER", "ASSISTANT"),
256
+ version="v1",
257
+ messages=(),
258
+ offset=0,
259
+ sep_style=SeparatorStyle.TWO,
260
+ sep=" ",
261
+ sep2="</s>",
262
+ )
263
+
264
+ conv_llama_2 = Conversation(
265
+ system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
266
+
267
+ If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
268
+ roles=("USER", "ASSISTANT"),
269
+ version="llama_v2",
270
+ messages=(),
271
+ offset=0,
272
+ sep_style=SeparatorStyle.LLAMA_2,
273
+ sep="<s>",
274
+ sep2="</s>",
275
+ )
276
+
277
+ conv_llava_llama_2 = Conversation(
278
+ system="You are a helpful language and vision assistant. "
279
+ "You are able to understand the visual content that the user provides, "
280
+ "and assist the user with a variety of tasks using natural language.",
281
+ roles=("USER", "ASSISTANT"),
282
+ version="llama_v2",
283
+ messages=(),
284
+ offset=0,
285
+ sep_style=SeparatorStyle.LLAMA_2,
286
+ sep="<s>",
287
+ sep2="</s>",
288
+ )
289
+
290
+ conv_mpt = Conversation(
291
+ system="""<|im_start|>system
292
+ A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
293
+ roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
294
+ version="mpt",
295
+ messages=(),
296
+ offset=0,
297
+ sep_style=SeparatorStyle.MPT,
298
+ sep="<|im_end|>",
299
+ )
300
+
301
+ conv_llava_plain = Conversation(
302
+ system="",
303
+ roles=("", ""),
304
+ messages=(
305
+ ),
306
+ offset=0,
307
+ sep_style=SeparatorStyle.PLAIN,
308
+ sep="\n",
309
+ )
310
+
311
+ conv_llava_v0 = Conversation(
312
+ system="A chat between a curious human and an artificial intelligence assistant. "
313
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.",
314
+ roles=("Human", "Assistant"),
315
+ messages=(
316
+ ),
317
+ offset=0,
318
+ sep_style=SeparatorStyle.SINGLE,
319
+ sep="###",
320
+ )
321
+
322
+ conv_llava_v0_mmtag = Conversation(
323
+ system="A chat between a curious user and an artificial intelligence assistant. "
324
+ "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
325
+ "The visual content will be provided with the following format: <Image>visual content</Image>.",
326
+ roles=("Human", "Assistant"),
327
+ messages=(
328
+ ),
329
+ offset=0,
330
+ sep_style=SeparatorStyle.SINGLE,
331
+ sep="###",
332
+ version="v0_mmtag",
333
+ )
334
+
335
+ conv_llava_v1 = Conversation(
336
+ system="A chat between a curious human and an artificial intelligence assistant. "
337
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.",
338
+ roles=("USER", "ASSISTANT"),
339
+ version="v1",
340
+ messages=(),
341
+ offset=0,
342
+ sep_style=SeparatorStyle.TWO,
343
+ sep=" ",
344
+ sep2="</s>",
345
+ )
346
+
347
+ conv_llava_v1_mmtag = Conversation(
348
+ system="A chat between a curious user and an artificial intelligence assistant. "
349
+ "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
350
+ "The visual content will be provided with the following format: <Image>visual content</Image>.",
351
+ roles=("USER", "ASSISTANT"),
352
+ messages=(),
353
+ offset=0,
354
+ sep_style=SeparatorStyle.TWO,
355
+ sep=" ",
356
+ sep2="</s>",
357
+ version="v1_mmtag",
358
+ )
359
+
360
+ default_conversation = conv_vicuna_v0
361
+ conv_templates = {
362
+ "default": conv_vicuna_v0,
363
+ "v0": conv_vicuna_v0,
364
+ "v1": conv_vicuna_v1,
365
+ "vicuna_v1": conv_vicuna_v1,
366
+ "llama_2": conv_llama_2,
367
+
368
+ "plain": conv_llava_plain,
369
+ "v0_plain": conv_llava_plain,
370
+ "llava_v0": conv_llava_v0,
371
+ "v0_mmtag": conv_llava_v0_mmtag,
372
+ "llava_v1": conv_llava_v1,
373
+ "v1_mmtag": conv_llava_v1_mmtag,
374
+ "llava_llama_2": conv_llava_llama_2,
375
+
376
+ "mpt": conv_mpt,
377
+ }
378
+
379
+
380
+ if __name__ == "__main__":
381
+ print(default_conversation.get_prompt())
llava/eval/eval_gpt_review.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+
5
+ import openai
6
+ import tqdm
7
+ import ray
8
+ import time
9
+
10
+ NUM_SECONDS_TO_SLEEP = 3
11
+
12
+ @ray.remote(num_cpus=4)
13
+ def get_eval(content: str, max_tokens: int):
14
+ while True:
15
+ try:
16
+ response = openai.ChatCompletion.create(
17
+ model='gpt-4',
18
+ messages=[{
19
+ 'role': 'system',
20
+ 'content': 'You are a helpful and precise assistant for checking the quality of the answer.'
21
+ }, {
22
+ 'role': 'user',
23
+ 'content': content,
24
+ }],
25
+ temperature=0.2, # TODO: figure out which temperature is best for evaluation
26
+ max_tokens=max_tokens,
27
+ )
28
+ break
29
+ except openai.error.RateLimitError:
30
+ pass
31
+ except Exception as e:
32
+ print(e)
33
+ time.sleep(NUM_SECONDS_TO_SLEEP)
34
+
35
+ print('success!')
36
+ return response['choices'][0]['message']['content']
37
+
38
+
39
+ def parse_score(review):
40
+ try:
41
+ score_pair = review.split('\n')[0]
42
+ score_pair = score_pair.replace(',', ' ')
43
+ sp = score_pair.split(' ')
44
+ if len(sp) == 2:
45
+ return [float(sp[0]), float(sp[1])]
46
+ else:
47
+ print('error', review)
48
+ return [-1, -1]
49
+ except Exception as e:
50
+ print(e)
51
+ print('error', review)
52
+ return [-1, -1]
53
+
54
+
55
+ if __name__ == '__main__':
56
+ parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
57
+ parser.add_argument('-q', '--question')
58
+ # parser.add_argument('-a', '--answer')
59
+ parser.add_argument('-a', '--answer-list', nargs='+', default=[])
60
+ parser.add_argument('-r', '--rule')
61
+ parser.add_argument('-o', '--output')
62
+ parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output')
63
+ args = parser.parse_args()
64
+
65
+ ray.init()
66
+
67
+ f_q = open(os.path.expanduser(args.question))
68
+ f_ans1 = open(os.path.expanduser(args.answer_list[0]))
69
+ f_ans2 = open(os.path.expanduser(args.answer_list[1]))
70
+ rule_dict = json.load(open(os.path.expanduser(args.rule), 'r'))
71
+
72
+ review_file = open(f'{args.output}', 'w')
73
+
74
+ js_list = []
75
+ handles = []
76
+ idx = 0
77
+ for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2):
78
+ # if idx == 1:
79
+ # break
80
+
81
+ ques = json.loads(ques_js)
82
+ ans1 = json.loads(ans1_js)
83
+ ans2 = json.loads(ans2_js)
84
+
85
+ category = json.loads(ques_js)['category']
86
+ if category in rule_dict:
87
+ rule = rule_dict[category]
88
+ else:
89
+ rule = rule_dict['default']
90
+ prompt = rule['prompt']
91
+ role = rule['role']
92
+ content = (f'[Question]\n{ques["text"]}\n\n'
93
+ f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n'
94
+ f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n'
95
+ f'[System]\n{prompt}\n\n')
96
+ js_list.append({
97
+ 'id': idx+1,
98
+ 'question_id': ques['question_id'],
99
+ 'answer1_id': ans1['answer_id'],
100
+ 'answer2_id': ans2['answer_id'],
101
+ 'category': category})
102
+ idx += 1
103
+ handles.append(get_eval.remote(content, args.max_tokens))
104
+ # To avoid the rate limit set by OpenAI
105
+ time.sleep(NUM_SECONDS_TO_SLEEP)
106
+
107
+ reviews = ray.get(handles)
108
+ for idx, review in enumerate(reviews):
109
+ scores = parse_score(review)
110
+ js_list[idx]['content'] = review
111
+ js_list[idx]['tuple'] = scores
112
+ review_file.write(json.dumps(js_list[idx]) + '\n')
113
+ review_file.close()
llava/eval/eval_gpt_review_bench.py ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+
5
+ import openai
6
+ import time
7
+
8
+ NUM_SECONDS_TO_SLEEP = 0.5
9
+
10
+
11
+ def get_eval(content: str, max_tokens: int):
12
+ while True:
13
+ try:
14
+ response = openai.ChatCompletion.create(
15
+ model='gpt-4-0314',
16
+ messages=[{
17
+ 'role': 'system',
18
+ 'content': 'You are a helpful and precise assistant for checking the quality of the answer.'
19
+ }, {
20
+ 'role': 'user',
21
+ 'content': content,
22
+ }],
23
+ temperature=0.2, # TODO: figure out which temperature is best for evaluation
24
+ max_tokens=max_tokens,
25
+ )
26
+ break
27
+ except openai.error.RateLimitError:
28
+ pass
29
+ except Exception as e:
30
+ print(e)
31
+ time.sleep(NUM_SECONDS_TO_SLEEP)
32
+
33
+ return response['choices'][0]['message']['content']
34
+
35
+
36
+ def parse_score(review):
37
+ try:
38
+ score_pair = review.split('\n')[0]
39
+ score_pair = score_pair.replace(',', ' ')
40
+ sp = score_pair.split(' ')
41
+ if len(sp) == 2:
42
+ return [float(sp[0]), float(sp[1])]
43
+ else:
44
+ print('error', review)
45
+ return [-1, -1]
46
+ except Exception as e:
47
+ print(e)
48
+ print('error', review)
49
+ return [-1, -1]
50
+
51
+
52
+ if __name__ == '__main__':
53
+ parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
54
+ parser.add_argument('-q', '--question')
55
+ parser.add_argument('-c', '--context')
56
+ parser.add_argument('-a', '--answer-list', nargs='+', default=[])
57
+ parser.add_argument('-r', '--rule')
58
+ parser.add_argument('-o', '--output')
59
+ parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output')
60
+ args = parser.parse_args()
61
+
62
+ f_q = open(os.path.expanduser(args.question))
63
+ f_ans1 = open(os.path.expanduser(args.answer_list[0]))
64
+ f_ans2 = open(os.path.expanduser(args.answer_list[1]))
65
+ rule_dict = json.load(open(os.path.expanduser(args.rule), 'r'))
66
+
67
+ if os.path.isfile(os.path.expanduser(args.output)):
68
+ cur_reviews = [json.loads(line) for line in open(os.path.expanduser(args.output))]
69
+ else:
70
+ cur_reviews = []
71
+
72
+ review_file = open(f'{args.output}', 'a')
73
+
74
+ context_list = [json.loads(line) for line in open(os.path.expanduser(args.context))]
75
+ image_to_context = {context['image']: context for context in context_list}
76
+
77
+ handles = []
78
+ idx = 0
79
+ for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2):
80
+ ques = json.loads(ques_js)
81
+ ans1 = json.loads(ans1_js)
82
+ ans2 = json.loads(ans2_js)
83
+
84
+ inst = image_to_context[ques['image']]
85
+
86
+ if isinstance(inst['caption'], list):
87
+ cap_str = '\n'.join(inst['caption'])
88
+ else:
89
+ cap_str = inst['caption']
90
+
91
+ category = 'llava_bench_' + json.loads(ques_js)['category']
92
+ if category in rule_dict:
93
+ rule = rule_dict[category]
94
+ else:
95
+ assert False, f"Visual QA category not found in rule file: {category}."
96
+ prompt = rule['prompt']
97
+ role = rule['role']
98
+ content = (f'[Context]\n{cap_str}\n\n'
99
+ f'[Question]\n{ques["text"]}\n\n'
100
+ f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n'
101
+ f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n'
102
+ f'[System]\n{prompt}\n\n')
103
+ cur_js = {
104
+ 'id': idx+1,
105
+ 'question_id': ques['question_id'],
106
+ 'answer1_id': ans1.get('answer_id', ans1['question_id']),
107
+ 'answer2_id': ans2.get('answer_id', ans2['answer_id']),
108
+ 'category': category
109
+ }
110
+ if idx >= len(cur_reviews):
111
+ review = get_eval(content, args.max_tokens)
112
+ scores = parse_score(review)
113
+ cur_js['content'] = review
114
+ cur_js['tuple'] = scores
115
+ review_file.write(json.dumps(cur_js) + '\n')
116
+ review_file.flush()
117
+ else:
118
+ print(f'Skipping {idx} as we already have it.')
119
+ idx += 1
120
+ print(idx)
121
+ review_file.close()
llava/eval/eval_gpt_review_visual.py ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+
5
+ import openai
6
+ import time
7
+
8
+ NUM_SECONDS_TO_SLEEP = 0.5
9
+
10
+
11
+ def get_eval(content: str, max_tokens: int):
12
+ while True:
13
+ try:
14
+ response = openai.ChatCompletion.create(
15
+ model='gpt-4-0314',
16
+ messages=[{
17
+ 'role': 'system',
18
+ 'content': 'You are a helpful and precise assistant for checking the quality of the answer.'
19
+ }, {
20
+ 'role': 'user',
21
+ 'content': content,
22
+ }],
23
+ temperature=0.2, # TODO: figure out which temperature is best for evaluation
24
+ max_tokens=max_tokens,
25
+ )
26
+ break
27
+ except openai.error.RateLimitError:
28
+ pass
29
+ except Exception as e:
30
+ print(e)
31
+ time.sleep(NUM_SECONDS_TO_SLEEP)
32
+
33
+ return response['choices'][0]['message']['content']
34
+
35
+
36
+ def parse_score(review):
37
+ try:
38
+ score_pair = review.split('\n')[0]
39
+ score_pair = score_pair.replace(',', ' ')
40
+ sp = score_pair.split(' ')
41
+ if len(sp) == 2:
42
+ return [float(sp[0]), float(sp[1])]
43
+ else:
44
+ print('error', review)
45
+ return [-1, -1]
46
+ except Exception as e:
47
+ print(e)
48
+ print('error', review)
49
+ return [-1, -1]
50
+
51
+
52
+ if __name__ == '__main__':
53
+ parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
54
+ parser.add_argument('-q', '--question')
55
+ parser.add_argument('-c', '--context')
56
+ parser.add_argument('-a', '--answer-list', nargs='+', default=[])
57
+ parser.add_argument('-r', '--rule')
58
+ parser.add_argument('-o', '--output')
59
+ parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output')
60
+ args = parser.parse_args()
61
+
62
+ f_q = open(os.path.expanduser(args.question))
63
+ f_ans1 = open(os.path.expanduser(args.answer_list[0]))
64
+ f_ans2 = open(os.path.expanduser(args.answer_list[1]))
65
+ rule_dict = json.load(open(os.path.expanduser(args.rule), 'r'))
66
+
67
+ if os.path.isfile(os.path.expanduser(args.output)):
68
+ cur_reviews = [json.loads(line) for line in open(os.path.expanduser(args.output))]
69
+ else:
70
+ cur_reviews = []
71
+
72
+ review_file = open(f'{args.output}', 'a')
73
+
74
+ context_list = [json.loads(line) for line in open(os.path.expanduser(args.context))]
75
+ image_to_context = {context['image']: context for context in context_list}
76
+
77
+ handles = []
78
+ idx = 0
79
+ for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2):
80
+ ques = json.loads(ques_js)
81
+ ans1 = json.loads(ans1_js)
82
+ ans2 = json.loads(ans2_js)
83
+
84
+ inst = image_to_context[ques['image']]
85
+ cap_str = '\n'.join(inst['captions'])
86
+ box_str = '\n'.join([f'{instance["category"]}: {instance["bbox"]}' for instance in inst['instances']])
87
+
88
+ category = json.loads(ques_js)['category']
89
+ if category in rule_dict:
90
+ rule = rule_dict[category]
91
+ else:
92
+ assert False, f"Visual QA category not found in rule file: {category}."
93
+ prompt = rule['prompt']
94
+ role = rule['role']
95
+ content = (f'[Context]\n{cap_str}\n\n{box_str}\n\n'
96
+ f'[Question]\n{ques["text"]}\n\n'
97
+ f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n'
98
+ f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n'
99
+ f'[System]\n{prompt}\n\n')
100
+ cur_js = {
101
+ 'id': idx+1,
102
+ 'question_id': ques['question_id'],
103
+ 'answer1_id': ans1.get('answer_id', ans1['question_id']),
104
+ 'answer2_id': ans2.get('answer_id', ans2['answer_id']),
105
+ 'category': category
106
+ }
107
+ if idx >= len(cur_reviews):
108
+ review = get_eval(content, args.max_tokens)
109
+ scores = parse_score(review)
110
+ cur_js['content'] = review
111
+ cur_js['tuple'] = scores
112
+ review_file.write(json.dumps(cur_js) + '\n')
113
+ review_file.flush()
114
+ else:
115
+ print(f'Skipping {idx} as we already have it.')
116
+ idx += 1
117
+ print(idx)
118
+ review_file.close()
llava/eval/eval_science_qa.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+ import re
5
+ import random
6
+
7
+
8
+ def get_args():
9
+ parser = argparse.ArgumentParser()
10
+ parser.add_argument('--base-dir', type=str)
11
+ parser.add_argument('--result-file', type=str)
12
+ parser.add_argument('--output-file', type=str)
13
+ parser.add_argument('--output-result', type=str)
14
+ parser.add_argument('--split', type=str, default='test')
15
+ parser.add_argument('--options', type=list, default=["A", "B", "C", "D", "E"])
16
+ return parser.parse_args()
17
+
18
+
19
+ def convert_caps(results):
20
+ fakecaps = []
21
+ for result in results:
22
+ image_id = result['question_id']
23
+ caption = result['text']
24
+ fakecaps.append({"image_id": int(image_id), "caption": caption})
25
+ return fakecaps
26
+
27
+
28
+ def get_pred_idx(prediction, choices, options):
29
+ """
30
+ Get the index (e.g. 2) from the prediction (e.g. 'C')
31
+ """
32
+ if prediction in options[:len(choices)]:
33
+ return options.index(prediction)
34
+ else:
35
+ return random.choice(range(len(choices)))
36
+
37
+
38
+ if __name__ == "__main__":
39
+ args = get_args()
40
+
41
+ base_dir = args.base_dir
42
+ split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[args.split]
43
+ problems = json.load(open(os.path.join(base_dir, "problems.json")))
44
+ predictions = [json.loads(line) for line in open(args.result_file)]
45
+ predictions = {pred['question_id']: pred for pred in predictions}
46
+ split_problems = {idx: problems[idx] for idx in split_indices}
47
+
48
+ results = {'correct': [], 'incorrect': []}
49
+ sqa_results = {}
50
+ sqa_results['acc'] = None
51
+ sqa_results['correct'] = None
52
+ sqa_results['count'] = None
53
+ sqa_results['results'] = {}
54
+ sqa_results['outputs'] = {}
55
+
56
+ for prob_id, prob in split_problems.items():
57
+ if prob_id not in predictions:
58
+ continue
59
+ pred = predictions[prob_id]
60
+ pred_text = pred['text']
61
+
62
+ pattern = re.compile(r'The answer is ([A-Z]).')
63
+ res = pattern.findall(pred_text)
64
+ if len(res) == 1:
65
+ answer = res[0] # 'A', 'B', ...
66
+ else:
67
+ answer = "FAILED"
68
+
69
+ pred_idx = get_pred_idx(answer, prob['choices'], args.options)
70
+
71
+ analysis = {
72
+ 'question_id': prob_id,
73
+ 'parsed_ans': answer,
74
+ 'ground_truth': args.options[prob['answer']],
75
+ 'question': pred['prompt'],
76
+ 'pred': pred_text,
77
+ 'is_multimodal': '<image>' in pred['prompt'],
78
+ }
79
+
80
+ sqa_results['results'][prob_id] = get_pred_idx(answer, prob['choices'], args.options)
81
+ sqa_results['outputs'][prob_id] = pred_text
82
+
83
+ if pred_idx == prob['answer']:
84
+ results['correct'].append(analysis)
85
+ else:
86
+ results['incorrect'].append(analysis)
87
+
88
+ correct = len(results['correct'])
89
+ total = len(results['correct']) + len(results['incorrect'])
90
+ print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%')
91
+
92
+ sqa_results['acc'] = correct / total * 100
93
+ sqa_results['correct'] = correct
94
+ sqa_results['count'] = total
95
+
96
+ with open(args.output_file, 'w') as f:
97
+ json.dump(results, f, indent=2)
98
+ with open(args.output_result, 'w') as f:
99
+ json.dump(sqa_results, f, indent=2)
llava/eval/eval_science_qa_gpt4.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+ import re
5
+ import random
6
+ from collections import defaultdict
7
+
8
+
9
+ def get_args():
10
+ parser = argparse.ArgumentParser()
11
+ parser.add_argument('--base-dir', type=str)
12
+ parser.add_argument('--gpt4-result', type=str)
13
+ parser.add_argument('--our-result', type=str)
14
+ parser.add_argument('--split', type=str, default='test')
15
+ parser.add_argument('--options', type=list, default=["A", "B", "C", "D", "E"])
16
+ return parser.parse_args()
17
+
18
+
19
+ def convert_caps(results):
20
+ fakecaps = []
21
+ for result in results:
22
+ image_id = result['question_id']
23
+ caption = result['text']
24
+ fakecaps.append({"image_id": int(image_id), "caption": caption})
25
+ return fakecaps
26
+
27
+
28
+ def get_pred_idx(prediction, choices, options):
29
+ """
30
+ Get the index (e.g. 2) from the prediction (e.g. 'C')
31
+ """
32
+ if prediction in options[:len(choices)]:
33
+ return options.index(prediction)
34
+ else:
35
+ return random.choice(range(len(choices)))
36
+
37
+
38
+ if __name__ == "__main__":
39
+ args = get_args()
40
+
41
+ base_dir = args.base_dir
42
+ split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[args.split]
43
+ problems = json.load(open(os.path.join(base_dir, "problems.json")))
44
+ our_predictions = [json.loads(line) for line in open(args.our_result)]
45
+ our_predictions = {pred['question_id']: pred for pred in our_predictions}
46
+ split_problems = {idx: problems[idx] for idx in split_indices}
47
+
48
+ gpt4_predictions = json.load(open(args.gpt4_result))['outputs']
49
+
50
+ results = defaultdict(lambda: 0)
51
+
52
+ for prob_id, prob in split_problems.items():
53
+ if prob_id not in our_predictions:
54
+ continue
55
+ if prob_id not in gpt4_predictions:
56
+ continue
57
+ our_pred = our_predictions[prob_id]['text']
58
+ gpt4_pred = gpt4_predictions[prob_id]
59
+
60
+ pattern = re.compile(r'The answer is ([A-Z]).')
61
+ our_res = pattern.findall(our_pred)
62
+ if len(our_res) == 1:
63
+ our_answer = our_res[0] # 'A', 'B', ...
64
+ else:
65
+ our_answer = "FAILED"
66
+ gpt4_res = pattern.findall(gpt4_pred)
67
+ if len(gpt4_res) == 1:
68
+ gpt4_answer = gpt4_res[0] # 'A', 'B', ...
69
+ else:
70
+ gpt4_answer = "FAILED"
71
+
72
+ our_pred_idx = get_pred_idx(our_answer, prob['choices'], args.options)
73
+ gpt4_pred_idx = get_pred_idx(gpt4_answer, prob['choices'], args.options)
74
+
75
+ if gpt4_answer == 'FAILED':
76
+ results['gpt4_failed'] += 1
77
+ # continue
78
+ gpt4_pred_idx = our_pred_idx
79
+ # if our_pred_idx != prob['answer']:
80
+ # print(our_predictions[prob_id]['prompt'])
81
+ # print('-----------------')
82
+ # print(f'LECTURE: {prob["lecture"]}')
83
+ # print(f'SOLUTION: {prob["solution"]}')
84
+ # print('=====================')
85
+ else:
86
+ # continue
87
+ pass
88
+ # gpt4_pred_idx = our_pred_idx
89
+
90
+ if gpt4_pred_idx == prob['answer']:
91
+ results['correct'] += 1
92
+ else:
93
+ results['incorrect'] += 1
94
+
95
+
96
+ if gpt4_pred_idx == prob['answer'] or our_pred_idx == prob['answer']:
97
+ results['correct_upperbound'] += 1
98
+
99
+ correct = results['correct']
100
+ total = results['correct'] + results['incorrect']
101
+ print(f'Total: {total}, Correct: {correct}, Accuracy: {correct / total * 100:.2f}%')
102
+ print(f'Total: {total}, Correct (upper): {results["correct_upperbound"]}, Accuracy: {results["correct_upperbound"] / total * 100:.2f}%')
103
+ print(f'Total: {total}, GPT-4 NO-ANS (RANDOM): {results["gpt4_failed"]}, Percentage: {results["gpt4_failed"] / total * 100:.2f}%')
104
+
llava/eval/eval_science_qa_gpt4_requery.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import os
4
+ import re
5
+ import random
6
+ from collections import defaultdict
7
+
8
+
9
+ def get_args():
10
+ parser = argparse.ArgumentParser()
11
+ parser.add_argument('--base-dir', type=str)
12
+ parser.add_argument('--gpt4-result', type=str)
13
+ parser.add_argument('--requery-result', type=str)
14
+ parser.add_argument('--our-result', type=str)
15
+ parser.add_argument('--output-result', type=str)
16
+ parser.add_argument('--split', type=str, default='test')
17
+ parser.add_argument('--options', type=list, default=["A", "B", "C", "D", "E"])
18
+ return parser.parse_args()
19
+
20
+
21
+ def convert_caps(results):
22
+ fakecaps = []
23
+ for result in results:
24
+ image_id = result['question_id']
25
+ caption = result['text']
26
+ fakecaps.append({"image_id": int(image_id), "caption": caption})
27
+ return fakecaps
28
+
29
+
30
+ def get_pred_idx(prediction, choices, options):
31
+ """
32
+ Get the index (e.g. 2) from the prediction (e.g. 'C')
33
+ """
34
+ if prediction in options[:len(choices)]:
35
+ return options.index(prediction)
36
+ else:
37
+ return random.choice(range(len(choices)))
38
+
39
+
40
+ if __name__ == "__main__":
41
+ args = get_args()
42
+
43
+ base_dir = args.base_dir
44
+ split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[args.split]
45
+ problems = json.load(open(os.path.join(base_dir, "problems.json")))
46
+ our_predictions = [json.loads(line) for line in open(args.our_result)]
47
+ our_predictions = {pred['question_id']: pred for pred in our_predictions}
48
+ split_problems = {idx: problems[idx] for idx in split_indices}
49
+
50
+ requery_predictions = [json.loads(line) for line in open(args.requery_result)]
51
+ requery_predictions = {pred['question_id']: pred for pred in requery_predictions}
52
+
53
+ gpt4_predictions = json.load(open(args.gpt4_result))['outputs']
54
+
55
+ results = defaultdict(lambda: 0)
56
+
57
+ sqa_results = {}
58
+ sqa_results['acc'] = None
59
+ sqa_results['correct'] = None
60
+ sqa_results['count'] = None
61
+ sqa_results['results'] = {}
62
+ sqa_results['outputs'] = {}
63
+
64
+ for prob_id, prob in split_problems.items():
65
+ if prob_id not in our_predictions:
66
+ assert False
67
+ if prob_id not in gpt4_predictions:
68
+ assert False
69
+ our_pred = our_predictions[prob_id]['text']
70
+ gpt4_pred = gpt4_predictions[prob_id]
71
+ if prob_id not in requery_predictions:
72
+ results['missing_requery'] += 1
73
+ requery_pred = "MISSING"
74
+ else:
75
+ requery_pred = requery_predictions[prob_id]['text']
76
+
77
+ pattern = re.compile(r'The answer is ([A-Z]).')
78
+ our_res = pattern.findall(our_pred)
79
+ if len(our_res) == 1:
80
+ our_answer = our_res[0] # 'A', 'B', ...
81
+ else:
82
+ our_answer = "FAILED"
83
+
84
+ requery_res = pattern.findall(requery_pred)
85
+ if len(requery_res) == 1:
86
+ requery_answer = requery_res[0] # 'A', 'B', ...
87
+ else:
88
+ requery_answer = "FAILED"
89
+
90
+ gpt4_res = pattern.findall(gpt4_pred)
91
+ if len(gpt4_res) == 1:
92
+ gpt4_answer = gpt4_res[0] # 'A', 'B', ...
93
+ else:
94
+ gpt4_answer = "FAILED"
95
+
96
+ our_pred_idx = get_pred_idx(our_answer, prob['choices'], args.options)
97
+ gpt4_pred_idx = get_pred_idx(gpt4_answer, prob['choices'], args.options)
98
+ requery_pred_idx = get_pred_idx(requery_answer, prob['choices'], args.options)
99
+
100
+ results['total'] += 1
101
+
102
+ if gpt4_answer == 'FAILED':
103
+ results['gpt4_failed'] += 1
104
+ if gpt4_pred_idx == prob['answer']:
105
+ results['gpt4_correct'] += 1
106
+ if our_pred_idx == prob['answer']:
107
+ results['gpt4_ourvisual_correct'] += 1
108
+ elif gpt4_pred_idx == prob['answer']:
109
+ results['gpt4_correct'] += 1
110
+ results['gpt4_ourvisual_correct'] += 1
111
+
112
+ if our_pred_idx == prob['answer']:
113
+ results['our_correct'] += 1
114
+
115
+ if requery_answer == 'FAILED':
116
+ sqa_results['results'][prob_id] = our_pred_idx
117
+ if our_pred_idx == prob['answer']:
118
+ results['requery_correct'] += 1
119
+ else:
120
+ sqa_results['results'][prob_id] = requery_pred_idx
121
+ if requery_pred_idx == prob['answer']:
122
+ results['requery_correct'] += 1
123
+ else:
124
+ print(f"""
125
+ Question ({args.options[prob['answer']]}): {our_predictions[prob_id]['prompt']}
126
+ Our ({our_answer}): {our_pred}
127
+ GPT-4 ({gpt4_answer}): {gpt4_pred}
128
+ Requery ({requery_answer}): {requery_pred}
129
+ print("=====================================")
130
+ """)
131
+
132
+ if gpt4_pred_idx == prob['answer'] or our_pred_idx == prob['answer']:
133
+ results['correct_upperbound'] += 1
134
+
135
+ total = results['total']
136
+ print(f'Total: {total}, Our-Correct: {results["our_correct"]}, Accuracy: {results["our_correct"] / total * 100:.2f}%')
137
+ print(f'Total: {total}, GPT-4-Correct: {results["gpt4_correct"]}, Accuracy: {results["gpt4_correct"] / total * 100:.2f}%')
138
+ print(f'Total: {total}, GPT-4 NO-ANS (RANDOM): {results["gpt4_failed"]}, Percentage: {results["gpt4_failed"] / total * 100:.2f}%')
139
+ print(f'Total: {total}, GPT-4-OursVisual-Correct: {results["gpt4_ourvisual_correct"]}, Accuracy: {results["gpt4_ourvisual_correct"] / total * 100:.2f}%')
140
+ print(f'Total: {total}, Requery-Correct: {results["requery_correct"]}, Accuracy: {results["requery_correct"] / total * 100:.2f}%')
141
+ print(f'Total: {total}, Correct upper: {results["correct_upperbound"]}, Accuracy: {results["correct_upperbound"] / total * 100:.2f}%')
142
+
143
+ sqa_results['acc'] = results["requery_correct"] / total * 100
144
+ sqa_results['correct'] = results["requery_correct"]
145
+ sqa_results['count'] = total
146
+
147
+ with open(args.output_result, 'w') as f:
148
+ json.dump(sqa_results, f, indent=2)
149
+
llava/eval/generate_webpage_data_from_table.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Generate json file for webpage."""
2
+ import json
3
+ import os
4
+ import re
5
+
6
+ # models = ['llama', 'alpaca', 'gpt35', 'bard']
7
+ models = ['vicuna']
8
+
9
+
10
+ def read_jsonl(path: str, key: str=None):
11
+ data = []
12
+ with open(os.path.expanduser(path)) as f:
13
+ for line in f:
14
+ if not line:
15
+ continue
16
+ data.append(json.loads(line))
17
+ if key is not None:
18
+ data.sort(key=lambda x: x[key])
19
+ data = {item[key]: item for item in data}
20
+ return data
21
+
22
+
23
+ def trim_hanging_lines(s: str, n: int) -> str:
24
+ s = s.strip()
25
+ for _ in range(n):
26
+ s = s.split('\n', 1)[1].strip()
27
+ return s
28
+
29
+
30
+ if __name__ == '__main__':
31
+ questions = read_jsonl('table/question.jsonl', key='question_id')
32
+
33
+ # alpaca_answers = read_jsonl('table/answer/answer_alpaca-13b.jsonl', key='question_id')
34
+ # bard_answers = read_jsonl('table/answer/answer_bard.jsonl', key='question_id')
35
+ # gpt35_answers = read_jsonl('table/answer/answer_gpt35.jsonl', key='question_id')
36
+ # llama_answers = read_jsonl('table/answer/answer_llama-13b.jsonl', key='question_id')
37
+ vicuna_answers = read_jsonl('table/answer/answer_vicuna-13b.jsonl', key='question_id')
38
+ ours_answers = read_jsonl('table/results/llama-13b-hf-alpaca.jsonl', key='question_id')
39
+
40
+ review_vicuna = read_jsonl('table/review/review_vicuna-13b_llama-13b-hf-alpaca.jsonl', key='question_id')
41
+ # review_alpaca = read_jsonl('table/review/review_alpaca-13b_vicuna-13b.jsonl', key='question_id')
42
+ # review_bard = read_jsonl('table/review/review_bard_vicuna-13b.jsonl', key='question_id')
43
+ # review_gpt35 = read_jsonl('table/review/review_gpt35_vicuna-13b.jsonl', key='question_id')
44
+ # review_llama = read_jsonl('table/review/review_llama-13b_vicuna-13b.jsonl', key='question_id')
45
+
46
+ records = []
47
+ for qid in questions.keys():
48
+ r = {
49
+ 'id': qid,
50
+ 'category': questions[qid]['category'],
51
+ 'question': questions[qid]['text'],
52
+ 'answers': {
53
+ # 'alpaca': alpaca_answers[qid]['text'],
54
+ # 'llama': llama_answers[qid]['text'],
55
+ # 'bard': bard_answers[qid]['text'],
56
+ # 'gpt35': gpt35_answers[qid]['text'],
57
+ 'vicuna': vicuna_answers[qid]['text'],
58
+ 'ours': ours_answers[qid]['text'],
59
+ },
60
+ 'evaluations': {
61
+ # 'alpaca': review_alpaca[qid]['text'],
62
+ # 'llama': review_llama[qid]['text'],
63
+ # 'bard': review_bard[qid]['text'],
64
+ 'vicuna': review_vicuna[qid]['content'],
65
+ # 'gpt35': review_gpt35[qid]['text'],
66
+ },
67
+ 'scores': {
68
+ 'vicuna': review_vicuna[qid]['tuple'],
69
+ # 'alpaca': review_alpaca[qid]['score'],
70
+ # 'llama': review_llama[qid]['score'],
71
+ # 'bard': review_bard[qid]['score'],
72
+ # 'gpt35': review_gpt35[qid]['score'],
73
+ },
74
+ }
75
+
76
+ # cleanup data
77
+ cleaned_evals = {}
78
+ for k, v in r['evaluations'].items():
79
+ v = v.strip()
80
+ lines = v.split('\n')
81
+ # trim the first line if it's a pair of numbers
82
+ if re.match(r'\d+[, ]+\d+', lines[0]):
83
+ lines = lines[1:]
84
+ v = '\n'.join(lines)
85
+ cleaned_evals[k] = v.replace('Assistant 1', "**Assistant 1**").replace('Assistant 2', '**Assistant 2**')
86
+
87
+ r['evaluations'] = cleaned_evals
88
+ records.append(r)
89
+
90
+ # Reorder the records, this is optional
91
+ for r in records:
92
+ if r['id'] <= 20:
93
+ r['id'] += 60
94
+ else:
95
+ r['id'] -= 20
96
+ for r in records:
97
+ if r['id'] <= 50:
98
+ r['id'] += 10
99
+ elif 50 < r['id'] <= 60:
100
+ r['id'] -= 50
101
+ for r in records:
102
+ if r['id'] == 7:
103
+ r['id'] = 1
104
+ elif r['id'] < 7:
105
+ r['id'] += 1
106
+
107
+ records.sort(key=lambda x: x['id'])
108
+
109
+ # Write to file
110
+ with open('webpage/data.json', 'w') as f:
111
+ json.dump({'questions': records, 'models': models}, f, indent=2)
llava/eval/model_qa.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ from transformers import AutoTokenizer, AutoModelForCausalLM, StoppingCriteria
3
+ import torch
4
+ import os
5
+ import json
6
+ from tqdm import tqdm
7
+ import shortuuid
8
+
9
+ from llava.conversation import default_conversation
10
+ from llava.utils import disable_torch_init
11
+
12
+
13
+ # new stopping implementation
14
+ class KeywordsStoppingCriteria(StoppingCriteria):
15
+ def __init__(self, keywords, tokenizer, input_ids):
16
+ self.keywords = keywords
17
+ self.tokenizer = tokenizer
18
+ self.start_len = None
19
+ self.input_ids = input_ids
20
+
21
+ def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
22
+ if self.start_len is None:
23
+ self.start_len = self.input_ids.shape[1]
24
+ else:
25
+ outputs = self.tokenizer.batch_decode(output_ids[:, self.start_len:], skip_special_tokens=True)[0]
26
+ for keyword in self.keywords:
27
+ if keyword in outputs:
28
+ return True
29
+ return False
30
+
31
+
32
+ @torch.inference_mode()
33
+ def eval_model(model_name, questions_file, answers_file):
34
+ # Model
35
+ disable_torch_init()
36
+ model_name = os.path.expanduser(model_name)
37
+ tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
38
+ model = AutoModelForCausalLM.from_pretrained(model_name,
39
+ torch_dtype=torch.float16).cuda()
40
+
41
+
42
+ ques_file = open(os.path.expanduser(questions_file), "r")
43
+ ans_file = open(os.path.expanduser(answers_file), "w")
44
+ for i, line in enumerate(tqdm(ques_file)):
45
+ idx = json.loads(line)["question_id"]
46
+ qs = json.loads(line)["text"]
47
+ cat = json.loads(line)["category"]
48
+ conv = default_conversation.copy()
49
+ conv.append_message(conv.roles[0], qs)
50
+ prompt = conv.get_prompt()
51
+ inputs = tokenizer([prompt])
52
+ input_ids = torch.as_tensor(inputs.input_ids).cuda()
53
+ stopping_criteria = KeywordsStoppingCriteria([conv.sep], tokenizer, input_ids)
54
+ output_ids = model.generate(
55
+ input_ids,
56
+ do_sample=True,
57
+ use_cache=True,
58
+ temperature=0.7,
59
+ max_new_tokens=1024,
60
+ stopping_criteria=[stopping_criteria])
61
+ outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
62
+ try:
63
+ index = outputs.index(conv.sep, len(prompt))
64
+ except ValueError:
65
+ outputs += conv.sep
66
+ index = outputs.index(conv.sep, len(prompt))
67
+
68
+ outputs = outputs[len(prompt) + len(conv.roles[1]) + 2:index].strip()
69
+ ans_id = shortuuid.uuid()
70
+ ans_file.write(json.dumps({"question_id": idx,
71
+ "text": outputs,
72
+ "answer_id": ans_id,
73
+ "model_id": model_name,
74
+ "metadata": {}}) + "\n")
75
+ ans_file.flush()
76
+ ans_file.close()
77
+
78
+ if __name__ == "__main__":
79
+ parser = argparse.ArgumentParser()
80
+ parser.add_argument("--model-name", type=str, default="facebook/opt-350m")
81
+ parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
82
+ parser.add_argument("--answers-file", type=str, default="answer.jsonl")
83
+ args = parser.parse_args()
84
+
85
+ eval_model(args.model_name, args.question_file, args.answers_file)
llava/eval/model_vqa.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import torch
3
+ import os
4
+ import json
5
+ from tqdm import tqdm
6
+ import shortuuid
7
+
8
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
9
+ from llava.conversation import conv_templates, SeparatorStyle
10
+ from llava.model.builder import load_pretrained_model
11
+ from llava.utils import disable_torch_init
12
+ from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
13
+
14
+ from PIL import Image
15
+ import math
16
+
17
+
18
+ def split_list(lst, n):
19
+ """Split a list into n (roughly) equal-sized chunks"""
20
+ chunk_size = math.ceil(len(lst) / n) # integer division
21
+ return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
22
+
23
+
24
+ def get_chunk(lst, n, k):
25
+ chunks = split_list(lst, n)
26
+ return chunks[k]
27
+
28
+
29
+ def eval_model(args):
30
+ # Model
31
+ disable_torch_init()
32
+ model_path = os.path.expanduser(args.model_path)
33
+ model_name = get_model_name_from_path(model_path)
34
+ tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name)
35
+
36
+ questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
37
+ questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
38
+ answers_file = os.path.expanduser(args.answers_file)
39
+ os.makedirs(os.path.dirname(answers_file), exist_ok=True)
40
+ ans_file = open(answers_file, "w")
41
+ for line in tqdm(questions):
42
+ idx = line["question_id"]
43
+ image_file = line["image"]
44
+ qs = line["text"]
45
+ cur_prompt = qs
46
+ if model.config.mm_use_im_start_end:
47
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
48
+ else:
49
+ qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
50
+
51
+ conv = conv_templates[args.conv_mode].copy()
52
+ conv.append_message(conv.roles[0], qs)
53
+ conv.append_message(conv.roles[1], None)
54
+ prompt = conv.get_prompt()
55
+
56
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
57
+
58
+ image = Image.open(os.path.join(args.image_folder, image_file))
59
+ image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
60
+
61
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
62
+ keywords = [stop_str]
63
+ stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
64
+
65
+ with torch.inference_mode():
66
+ output_ids = model.generate(
67
+ input_ids,
68
+ images=image_tensor.unsqueeze(0).half().cuda(),
69
+ do_sample=True,
70
+ temperature=args.temperature,
71
+ top_p=args.top_p,
72
+ num_beams=args.num_beams,
73
+ # no_repeat_ngram_size=3,
74
+ max_new_tokens=1024,
75
+ use_cache=True)
76
+
77
+ input_token_len = input_ids.shape[1]
78
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
79
+ if n_diff_input_output > 0:
80
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
81
+ outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
82
+ outputs = outputs.strip()
83
+ if outputs.endswith(stop_str):
84
+ outputs = outputs[:-len(stop_str)]
85
+ outputs = outputs.strip()
86
+
87
+ ans_id = shortuuid.uuid()
88
+ ans_file.write(json.dumps({"question_id": idx,
89
+ "prompt": cur_prompt,
90
+ "text": outputs,
91
+ "answer_id": ans_id,
92
+ "model_id": model_name,
93
+ "metadata": {}}) + "\n")
94
+ ans_file.flush()
95
+ ans_file.close()
96
+
97
+ if __name__ == "__main__":
98
+ parser = argparse.ArgumentParser()
99
+ parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
100
+ parser.add_argument("--model-base", type=str, default=None)
101
+ parser.add_argument("--image-folder", type=str, default="")
102
+ parser.add_argument("--question-file", type=str, default="tables/question.jsonl")
103
+ parser.add_argument("--answers-file", type=str, default="answer.jsonl")
104
+ parser.add_argument("--conv-mode", type=str, default="llava_v1")
105
+ parser.add_argument("--num-chunks", type=int, default=1)
106
+ parser.add_argument("--chunk-idx", type=int, default=0)
107
+ parser.add_argument("--temperature", type=float, default=0.2)
108
+ parser.add_argument("--top_p", type=float, default=None)
109
+ parser.add_argument("--num_beams", type=int, default=1)
110
+ args = parser.parse_args()
111
+
112
+ eval_model(args)
llava/eval/model_vqa_science.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import torch
3
+ import os
4
+ import json
5
+ from tqdm import tqdm
6
+ import shortuuid
7
+
8
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
9
+ from llava.conversation import conv_templates, SeparatorStyle
10
+ from llava.model.builder import load_pretrained_model
11
+ from llava.utils import disable_torch_init
12
+ from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
13
+
14
+ from PIL import Image
15
+ import math
16
+
17
+
18
+ def split_list(lst, n):
19
+ """Split a list into n (roughly) equal-sized chunks"""
20
+ chunk_size = math.ceil(len(lst) / n) # integer division
21
+ return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
22
+
23
+
24
+ def get_chunk(lst, n, k):
25
+ chunks = split_list(lst, n)
26
+ return chunks[k]
27
+
28
+
29
+ def eval_model(args):
30
+ # Model
31
+ disable_torch_init()
32
+ model_path = os.path.expanduser(args.model_path)
33
+ model_name = get_model_name_from_path(model_path)
34
+ tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name)
35
+
36
+ questions = json.load(open(os.path.expanduser(args.question_file), "r"))
37
+ questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
38
+ answers_file = os.path.expanduser(args.answers_file)
39
+ os.makedirs(os.path.dirname(answers_file), exist_ok=True)
40
+ ans_file = open(answers_file, "w")
41
+ for i, line in enumerate(tqdm(questions)):
42
+ idx = line["id"]
43
+ question = line['conversations'][0]
44
+ qs = question['value'].replace('<image>', '').strip()
45
+ cur_prompt = qs
46
+
47
+ if 'image' in line:
48
+ image_file = line["image"]
49
+ image = Image.open(os.path.join(args.image_folder, image_file))
50
+ image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
51
+ images = image_tensor.unsqueeze(0).half().cuda()
52
+ if getattr(model.config, 'mm_use_im_start_end', False):
53
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
54
+ else:
55
+ qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
56
+ cur_prompt = '<image>' + '\n' + cur_prompt
57
+ else:
58
+ images = None
59
+
60
+ conv = conv_templates[args.conv_mode].copy()
61
+ conv.append_message(conv.roles[0], qs)
62
+ conv.append_message(conv.roles[1], None)
63
+ prompt = conv.get_prompt()
64
+
65
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
66
+
67
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
68
+ keywords = [stop_str]
69
+ stopping_criteria = [KeywordsStoppingCriteria(keywords, tokenizer, input_ids)] if conv.version == "v0" else None
70
+
71
+ with torch.inference_mode():
72
+ output_ids = model.generate(
73
+ input_ids,
74
+ images=images,
75
+ do_sample=True,
76
+ temperature=0.2,
77
+ max_new_tokens=1024,
78
+ use_cache=True,
79
+ stopping_criteria=stopping_criteria,
80
+ )
81
+
82
+ input_token_len = input_ids.shape[1]
83
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
84
+ if n_diff_input_output > 0:
85
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
86
+ outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
87
+ outputs = outputs.strip()
88
+ if outputs.endswith(stop_str):
89
+ outputs = outputs[:-len(stop_str)]
90
+ outputs = outputs.strip()
91
+
92
+ # prompt for answer
93
+ if args.answer_prompter:
94
+ outputs_reasoning = outputs
95
+ input_ids = tokenizer_image_token(prompt + outputs_reasoning + ' ###\nANSWER:', tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
96
+
97
+ with torch.inference_mode():
98
+ output_ids = model.generate(
99
+ input_ids,
100
+ images=images,
101
+ do_sample=True,
102
+ temperature=0.2,
103
+ max_new_tokens=64,
104
+ use_cache=True,
105
+ stopping_criteria=[stopping_criteria])
106
+
107
+ input_token_len = input_ids.shape[1]
108
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
109
+ if n_diff_input_output > 0:
110
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
111
+ outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
112
+ outputs = outputs.strip()
113
+ if outputs.endswith(stop_str):
114
+ outputs = outputs[:-len(stop_str)]
115
+ outputs = outputs.strip()
116
+ outputs = outputs_reasoning + '\n The answer is ' + outputs
117
+
118
+ ans_id = shortuuid.uuid()
119
+ ans_file.write(json.dumps({"question_id": idx,
120
+ "prompt": cur_prompt,
121
+ "text": outputs,
122
+ "answer_id": ans_id,
123
+ "model_id": model_name,
124
+ "metadata": {}}) + "\n")
125
+ ans_file.flush()
126
+ ans_file.close()
127
+
128
+ if __name__ == "__main__":
129
+ parser = argparse.ArgumentParser()
130
+ parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
131
+ parser.add_argument("--model-base", type=str, default=None)
132
+ parser.add_argument("--image-folder", type=str, default="")
133
+ parser.add_argument("--question-file", type=str, default="tables/question.json")
134
+ parser.add_argument("--answers-file", type=str, default="answer.jsonl")
135
+ parser.add_argument("--conv-mode", type=str, default="llava_v0")
136
+ parser.add_argument("--num-chunks", type=int, default=1)
137
+ parser.add_argument("--chunk-idx", type=int, default=0)
138
+ parser.add_argument("--answer-prompter", action="store_true")
139
+ args = parser.parse_args()
140
+
141
+ eval_model(args)
llava/eval/qa_baseline_gpt35.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Generate answers with GPT-3.5"""
2
+ # Note: you need to be using OpenAI Python v0.27.0 for the code below to work
3
+ import argparse
4
+ import json
5
+ import os
6
+ import time
7
+ import concurrent.futures
8
+
9
+ import openai
10
+ import tqdm
11
+ import shortuuid
12
+
13
+ MODEL = 'gpt-3.5-turbo'
14
+ MODEL_ID = 'gpt-3.5-turbo:20230327'
15
+
16
+ def get_answer(question_id: int, question: str, max_tokens: int):
17
+ ans = {
18
+ 'answer_id': shortuuid.uuid(),
19
+ 'question_id': question_id,
20
+ 'model_id': MODEL_ID,
21
+ }
22
+ for _ in range(3):
23
+ try:
24
+ response = openai.ChatCompletion.create(
25
+ model=MODEL,
26
+ messages=[{
27
+ 'role': 'system',
28
+ 'content': 'You are a helpful assistant.'
29
+ }, {
30
+ 'role': 'user',
31
+ 'content': question,
32
+ }],
33
+ max_tokens=max_tokens,
34
+ )
35
+ ans['text'] = response['choices'][0]['message']['content']
36
+ return ans
37
+ except Exception as e:
38
+ print('[ERROR]', e)
39
+ ans['text'] = '#ERROR#'
40
+ time.sleep(1)
41
+ return ans
42
+
43
+
44
+ if __name__ == '__main__':
45
+ parser = argparse.ArgumentParser(description='ChatGPT answer generation.')
46
+ parser.add_argument('-q', '--question')
47
+ parser.add_argument('-o', '--output')
48
+ parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output')
49
+ args = parser.parse_args()
50
+
51
+ questions_dict = {}
52
+ with open(os.path.expanduser(args.question)) as f:
53
+ for line in f:
54
+ if not line:
55
+ continue
56
+ q = json.loads(line)
57
+ questions_dict[q['question_id']] = q['text']
58
+
59
+ answers = []
60
+
61
+ with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor:
62
+ futures = []
63
+ for qid, question in questions_dict.items():
64
+ future = executor.submit(get_answer, qid, question, args.max_tokens)
65
+ futures.append(future)
66
+
67
+ for future in tqdm.tqdm(concurrent.futures.as_completed(futures), total=len(futures)):
68
+ answers.append(future.result())
69
+
70
+ answers.sort(key=lambda x: x['question_id'])
71
+
72
+ with open(os.path.expanduser(args.output), 'w') as f:
73
+ table = [json.dumps(ans) for ans in answers]
74
+ f.write('\n'.join(table))
llava/eval/run_llava.py ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import torch
3
+
4
+ from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
5
+ from llava.conversation import conv_templates, SeparatorStyle
6
+ from llava.model.builder import load_pretrained_model
7
+ from llava.utils import disable_torch_init
8
+ from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
9
+
10
+ from PIL import Image
11
+
12
+ import requests
13
+ from PIL import Image
14
+ from io import BytesIO
15
+
16
+
17
+ def load_image(image_file):
18
+ if image_file.startswith('http') or image_file.startswith('https'):
19
+ response = requests.get(image_file)
20
+ image = Image.open(BytesIO(response.content)).convert('RGB')
21
+ else:
22
+ image = Image.open(image_file).convert('RGB')
23
+ return image
24
+
25
+
26
+ def eval_model(args):
27
+ # Model
28
+ disable_torch_init()
29
+
30
+ model_name = get_model_name_from_path(args.model_path)
31
+ tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name)
32
+
33
+ qs = args.query
34
+ if model.config.mm_use_im_start_end:
35
+ qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
36
+ else:
37
+ qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
38
+
39
+ if 'llama-2' in model_name.lower():
40
+ conv_mode = "llava_llama_2"
41
+ elif "v1" in model_name.lower():
42
+ conv_mode = "llava_v1"
43
+ elif "mpt" in model_name.lower():
44
+ conv_mode = "mpt"
45
+ else:
46
+ conv_mode = "llava_v0"
47
+
48
+ if args.conv_mode is not None and conv_mode != args.conv_mode:
49
+ print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, args.conv_mode, args.conv_mode))
50
+ else:
51
+ args.conv_mode = conv_mode
52
+
53
+ conv = conv_templates[args.conv_mode].copy()
54
+ conv.append_message(conv.roles[0], qs)
55
+ conv.append_message(conv.roles[1], None)
56
+ prompt = conv.get_prompt()
57
+
58
+ image = load_image(args.image_file)
59
+ image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].half().cuda()
60
+
61
+ input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
62
+
63
+ stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
64
+ keywords = [stop_str]
65
+ stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
66
+
67
+ with torch.inference_mode():
68
+ output_ids = model.generate(
69
+ input_ids,
70
+ images=image_tensor,
71
+ do_sample=True,
72
+ temperature=0.2,
73
+ max_new_tokens=1024,
74
+ use_cache=True,
75
+ stopping_criteria=[stopping_criteria])
76
+
77
+ input_token_len = input_ids.shape[1]
78
+ n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
79
+ if n_diff_input_output > 0:
80
+ print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
81
+ outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
82
+ outputs = outputs.strip()
83
+ if outputs.endswith(stop_str):
84
+ outputs = outputs[:-len(stop_str)]
85
+ outputs = outputs.strip()
86
+ print(outputs)
87
+
88
+ if __name__ == "__main__":
89
+ parser = argparse.ArgumentParser()
90
+ parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
91
+ parser.add_argument("--model-base", type=str, default=None)
92
+ parser.add_argument("--image-file", type=str, required=True)
93
+ parser.add_argument("--query", type=str, required=True)
94
+ parser.add_argument("--conv-mode", type=str, default=None)
95
+ args = parser.parse_args()
96
+
97
+ eval_model(args)
llava/eval/summarize_gpt_review.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ from collections import defaultdict
4
+
5
+ import numpy as np
6
+
7
+ import argparse
8
+
9
+ def parse_args():
10
+ parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
11
+ parser.add_argument('-d', '--dir', default=None)
12
+ parser.add_argument('-f', '--files', nargs='*', default=None)
13
+ parser.add_argument('-i', '--ignore', nargs='*', default=None)
14
+ return parser.parse_args()
15
+
16
+
17
+ if __name__ == '__main__':
18
+ args = parse_args()
19
+
20
+ if args.ignore is not None:
21
+ args.ignore = [int(x) for x in args.ignore]
22
+
23
+ if args.files is not None and len(args.files) > 0:
24
+ review_files = args.files
25
+ else:
26
+ review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_'))]
27
+
28
+ for review_file in sorted(review_files):
29
+ config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '')
30
+ scores = defaultdict(list)
31
+ print(config)
32
+ with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f:
33
+ for review_str in f:
34
+ review = json.loads(review_str)
35
+ if args.ignore is not None and review['question_id'] in args.ignore:
36
+ continue
37
+ if 'category' in review:
38
+ scores[review['category']].append(review['tuple'])
39
+ scores['all'].append(review['tuple'])
40
+ else:
41
+ if 'tuple' in review:
42
+ scores['all'].append(review['tuple'])
43
+ else:
44
+ scores['all'].append(review['score'])
45
+ for k, v in sorted(scores.items()):
46
+ stats = np.asarray(v).mean(0).tolist()
47
+ stats = [round(x, 3) for x in stats]
48
+ # print(k, stats, round(stats[1]/stats[0]*100, 1))
49
+ print(k, round(stats[1]/stats[0]*100, 1))
50
+ print('=================================')
llava/eval/webpage/figures/alpaca.png ADDED
llava/eval/webpage/figures/bard.jpg ADDED