LinB203 commited on
Commit
5244231
β€’
1 Parent(s): 0e023c7
Files changed (50) hide show
  1. scripts/convert_gqa_for_eval.py +18 -0
  2. scripts/convert_mmbench_for_submission.py +27 -0
  3. scripts/convert_mmvet_for_eval.py +18 -0
  4. scripts/convert_seed_for_submission.py +74 -0
  5. scripts/convert_sqa_to_llava.py +88 -0
  6. scripts/convert_sqa_to_llava_base_prompt.py +334 -0
  7. scripts/convert_vizwiz_for_submission.py +47 -0
  8. scripts/convert_vqav2_for_submission.py +56 -0
  9. scripts/eval_gpt_mmvet.py +276 -0
  10. scripts/finetune.sh +48 -0
  11. scripts/finetune_full_schedule.sh +48 -0
  12. scripts/finetune_lora.sh +49 -0
  13. scripts/finetune_qlora.sh +50 -0
  14. scripts/finetune_sqa.sh +36 -0
  15. scripts/merge_lora_weights.py +22 -0
  16. scripts/pretrain.sh +46 -0
  17. scripts/sqa_eval_batch.sh +13 -0
  18. scripts/sqa_eval_gather.sh +18 -0
  19. scripts/v1_5/eval/eval_benchmark_1_correctness.sh +17 -0
  20. scripts/v1_5/eval/eval_benchmark_2_detail.sh +18 -0
  21. scripts/v1_5/eval/eval_benchmark_3_contextual.sh +18 -0
  22. scripts/v1_5/eval/eval_benchmark_4_temporal.sh +18 -0
  23. scripts/v1_5/eval/eval_benchmark_5_consistency.sh +18 -0
  24. scripts/v1_5/eval/eval_image_gqa.sh +43 -0
  25. scripts/v1_5/eval/eval_image_llavabench.sh +24 -0
  26. scripts/v1_5/eval/eval_image_mmbench.sh +22 -0
  27. scripts/v1_5/eval/eval_image_mmvet.sh +24 -0
  28. scripts/v1_5/eval/eval_image_pope.sh +18 -0
  29. scripts/v1_5/eval/eval_image_sqa.sh +20 -0
  30. scripts/v1_5/eval/eval_image_textvqa.sh +17 -0
  31. scripts/v1_5/eval/eval_image_vizwiz.sh +17 -0
  32. scripts/v1_5/eval/eval_image_vqav2.sh +38 -0
  33. scripts/v1_5/eval/eval_qa_activitynet.sh +20 -0
  34. scripts/v1_5/eval/eval_qa_msrvtt.sh +20 -0
  35. scripts/v1_5/eval/eval_qa_msvd.sh +20 -0
  36. scripts/v1_5/eval/eval_qa_tgif.sh +20 -0
  37. scripts/v1_5/eval/run_benchmark_1_correctness.sh +18 -0
  38. scripts/v1_5/eval/run_benchmark_2_detail.sh +18 -0
  39. scripts/v1_5/eval/run_benchmark_3_contextual.sh +18 -0
  40. scripts/v1_5/eval/run_benchmark_4_temporal.sh +17 -0
  41. scripts/v1_5/eval/run_benchmark_5_consistency.sh +18 -0
  42. scripts/v1_5/eval/run_qa_activitynet.sh +42 -0
  43. scripts/v1_5/eval/run_qa_msrvtt.sh +43 -0
  44. scripts/v1_5/eval/run_qa_msvd.sh +42 -0
  45. scripts/v1_5/eval/run_qa_tgif.sh +42 -0
  46. scripts/v1_5/finetune.sh +42 -0
  47. scripts/v1_5/pretrain.sh +39 -0
  48. scripts/zero2.json +23 -0
  49. scripts/zero3.json +28 -0
  50. scripts/zero3_offload.json +56 -0
scripts/convert_gqa_for_eval.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import argparse
4
+
5
+ parser = argparse.ArgumentParser()
6
+ parser.add_argument("--src", type=str)
7
+ parser.add_argument("--dst", type=str)
8
+ args = parser.parse_args()
9
+
10
+ all_answers = []
11
+ for line_idx, line in enumerate(open(args.src)):
12
+ res = json.loads(line)
13
+ question_id = res['question_id']
14
+ text = res['text'].rstrip('.').lower()
15
+ all_answers.append({"questionId": question_id, "prediction": text})
16
+
17
+ with open(args.dst, 'w') as f:
18
+ json.dump(all_answers, f)
scripts/convert_mmbench_for_submission.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import argparse
4
+ import pandas as pd
5
+
6
+ def get_args():
7
+ parser = argparse.ArgumentParser()
8
+ parser.add_argument("--annotation-file", type=str, required=True)
9
+ parser.add_argument("--result-dir", type=str, required=True)
10
+ parser.add_argument("--upload-dir", type=str, required=True)
11
+ parser.add_argument("--experiment", type=str, required=True)
12
+
13
+ return parser.parse_args()
14
+
15
+ if __name__ == "__main__":
16
+ args = get_args()
17
+
18
+ df = pd.read_table(args.annotation_file)
19
+
20
+ cur_df = df.copy()
21
+ cur_df = cur_df.drop(columns=['hint', 'category', 'source', 'image', 'comment', 'l2-category'])
22
+ cur_df.insert(6, 'prediction', None)
23
+ for pred in open(os.path.join(args.result_dir, f"{args.experiment}.jsonl")):
24
+ pred = json.loads(pred)
25
+ cur_df.loc[df['index'] == pred['question_id'], 'prediction'] = pred['text']
26
+
27
+ cur_df.to_excel(os.path.join(args.upload_dir, f"{args.experiment}.xlsx"), index=False, engine='openpyxl')
scripts/convert_mmvet_for_eval.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import argparse
4
+
5
+ parser = argparse.ArgumentParser()
6
+ parser.add_argument("--src", type=str)
7
+ parser.add_argument("--dst", type=str)
8
+ args = parser.parse_args()
9
+
10
+ cur_result = {}
11
+
12
+ for line in open(args.src):
13
+ data = json.loads(line)
14
+ qid = data['question_id']
15
+ cur_result[f'v1_{qid}'] = data['text']
16
+
17
+ with open(args.dst, 'w') as f:
18
+ json.dump(cur_result, f, indent=2)
scripts/convert_seed_for_submission.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import argparse
4
+
5
+
6
+ def get_args():
7
+ parser = argparse.ArgumentParser()
8
+ parser.add_argument("--annotation-file", type=str)
9
+ parser.add_argument("--result-file", type=str)
10
+ parser.add_argument("--result-upload-file", type=str)
11
+ return parser.parse_args()
12
+
13
+
14
+ def eval_single(result_file, eval_only_type=None):
15
+ results = {}
16
+ for line in open(result_file):
17
+ row = json.loads(line)
18
+ results[row['question_id']] = row
19
+
20
+ type_counts = {}
21
+ correct_counts = {}
22
+ for question_data in data['questions']:
23
+ if eval_only_type is not None and question_data['data_type'] != eval_only_type: continue
24
+ data_type = question_data['question_type_id']
25
+ type_counts[data_type] = type_counts.get(data_type, 0) + 1
26
+ try:
27
+ question_id = int(question_data['question_id'])
28
+ except:
29
+ question_id = question_data['question_id']
30
+ if question_id not in results:
31
+ correct_counts[data_type] = correct_counts.get(data_type, 0)
32
+ continue
33
+ row = results[question_id]
34
+ if row['text'] == question_data['answer']:
35
+ correct_counts[data_type] = correct_counts.get(data_type, 0) + 1
36
+
37
+ total_count = 0
38
+ total_correct = 0
39
+ for data_type in sorted(type_counts.keys()):
40
+ accuracy = correct_counts[data_type] / type_counts[data_type] * 100
41
+ if eval_only_type is None:
42
+ print(f"{ques_type_id_to_name[data_type]}: {accuracy:.2f}%")
43
+
44
+ total_count += type_counts[data_type]
45
+ total_correct += correct_counts[data_type]
46
+
47
+ total_accuracy = total_correct / total_count * 100
48
+ if eval_only_type is None:
49
+ print(f"Total accuracy: {total_accuracy:.2f}%")
50
+ else:
51
+ print(f"{eval_only_type} accuracy: {total_accuracy:.2f}%")
52
+
53
+ return results
54
+
55
+ if __name__ == "__main__":
56
+ args = get_args()
57
+ data = json.load(open(args.annotation_file))
58
+ ques_type_id_to_name = {id:n for n,id in data['question_type'].items()}
59
+
60
+ results = eval_single(args.result_file)
61
+ eval_single(args.result_file, eval_only_type='image')
62
+ eval_single(args.result_file, eval_only_type='video')
63
+
64
+ with open(args.result_upload_file, 'w') as fp:
65
+ for question in data['questions']:
66
+ qid = question['question_id']
67
+ if qid in results:
68
+ result = results[qid]
69
+ else:
70
+ result = results[int(qid)]
71
+ fp.write(json.dumps({
72
+ 'question_id': qid,
73
+ 'prediction': result['text']
74
+ }) + '\n')
scripts/convert_sqa_to_llava.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import fire
4
+ import re
5
+ from convert_sqa_to_llava_base_prompt import build_prompt_chatbot
6
+
7
+
8
+ def convert_to_llava(base_dir, split, prompt_format="QCM-LEA"):
9
+ split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[split]
10
+ problems = json.load(open(os.path.join(base_dir, "problems.json")))
11
+
12
+ split_problems = build_prompt_chatbot(
13
+ problems, split_indices, prompt_format,
14
+ use_caption=False, is_test=False)
15
+
16
+ target_format = []
17
+ for prob_id, (input, output) in split_problems.items():
18
+ if input.startswith('Question: '):
19
+ input = input.replace('Question: ', '')
20
+ if output.startswith('Answer: '):
21
+ output = output.replace('Answer: ', '')
22
+
23
+ raw_prob_data = problems[prob_id]
24
+ if raw_prob_data['image'] is None:
25
+ target_format.append({
26
+ "id": prob_id,
27
+ "conversations": [
28
+ {'from': 'human', 'value': f"{input}"},
29
+ {'from': 'gpt', 'value': f"{output}"},
30
+ ],
31
+ })
32
+
33
+ else:
34
+ target_format.append({
35
+ "id": prob_id,
36
+ "image": os.path.join(prob_id, raw_prob_data['image']),
37
+ "conversations": [
38
+ {'from': 'human', 'value': f"{input}\n<image>"},
39
+ {'from': 'gpt', 'value': f"{output}"},
40
+ ],
41
+ })
42
+
43
+ print(f'Number of samples: {len(target_format)}')
44
+
45
+ with open(os.path.join(base_dir, f"llava_{split}_{prompt_format}.json"), "w") as f:
46
+ json.dump(target_format, f, indent=2)
47
+
48
+
49
+ def convert_to_jsonl(base_dir, split, prompt_format="QCM-LEPA"):
50
+ split_indices = json.load(open(os.path.join(base_dir, "pid_splits.json")))[split]
51
+ problems = json.load(open(os.path.join(base_dir, "problems.json")))
52
+
53
+ split_problems = build_prompt_chatbot(
54
+ problems, split_indices, prompt_format,
55
+ use_caption=False, is_test=False)
56
+
57
+ writer = open(os.path.join(base_dir, f"scienceqa_{split}_{prompt_format}.jsonl"), "w")
58
+ for prob_id, (input, output) in split_problems.items():
59
+ if input.startswith('Question: '):
60
+ input = input.replace('Question: ', '')
61
+ if output.startswith('Answer: '):
62
+ output = output.replace('Answer: ', '')
63
+
64
+ raw_prob_data = problems[prob_id]
65
+ if raw_prob_data['image'] is None:
66
+ data = {
67
+ "id": prob_id,
68
+ "instruction": f"{input}",
69
+ "output": f"{output}",
70
+ }
71
+
72
+ else:
73
+ data = {
74
+ "id": prob_id,
75
+ "image": os.path.join(prob_id, raw_prob_data['image']),
76
+ "instruction": f"{input}\n<image>",
77
+ "output": f"{output}",
78
+ }
79
+ writer.write(json.dumps(data) + '\n')
80
+ writer.close()
81
+
82
+
83
+ def main(task, **kwargs):
84
+ globals()[task](**kwargs)
85
+
86
+
87
+ if __name__ == "__main__":
88
+ fire.Fire(main)
scripts/convert_sqa_to_llava_base_prompt.py ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def get_question_text(problem):
2
+ question = problem['question']
3
+ return question
4
+
5
+
6
+ def get_context_text(problem, use_caption):
7
+ txt_context = problem['hint']
8
+ img_context = problem['caption'] if use_caption else ""
9
+ context = " ".join([txt_context, img_context]).strip()
10
+ if context == "":
11
+ context = "N/A"
12
+ return context
13
+
14
+
15
+ def get_choice_text(probelm, options):
16
+ choices = probelm['choices']
17
+ choice_list = []
18
+ for i, c in enumerate(choices):
19
+ choice_list.append("({}) {}".format(options[i], c))
20
+ choice_txt = " ".join(choice_list)
21
+ #print(choice_txt)
22
+ return choice_txt
23
+
24
+
25
+ def get_answer(problem, options):
26
+ return options[problem['answer']]
27
+
28
+
29
+ def get_lecture_text(problem):
30
+ # \\n: GPT-3 can generate the lecture with more tokens.
31
+ lecture = problem['lecture'].replace("\n", "\\n")
32
+ return lecture
33
+
34
+
35
+ def get_solution_text(problem):
36
+ # \\n: GPT-3 can generate the solution with more tokens
37
+ solution = problem['solution'].replace("\n", "\\n")
38
+ return solution
39
+
40
+
41
+ def create_one_example_chatbot(format, question, context, choice, answer, lecture, solution, test_example=True):
42
+
43
+ input_format, output_format = format.split("-")
44
+
45
+ ## Inputs
46
+ if input_format == "CQM":
47
+ input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
48
+ elif input_format == "QCM":
49
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
50
+ # upper bound experiment
51
+ elif input_format == "QCML":
52
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
53
+ elif input_format == "QCME":
54
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
55
+ elif input_format == "QCMLE":
56
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
57
+
58
+ elif input_format == "QCLM":
59
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
60
+ elif input_format == "QCEM":
61
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
62
+ elif input_format == "QCLEM":
63
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
64
+
65
+ # Outputs
66
+ if test_example:
67
+ output = "Answer:"
68
+ elif output_format == 'A':
69
+ output = f"Answer: The answer is {answer}."
70
+
71
+ elif output_format == 'AL':
72
+ output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
73
+ elif output_format == 'AE':
74
+ output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
75
+ elif output_format == 'ALE':
76
+ output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
77
+ elif output_format == 'AEL':
78
+ output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
79
+
80
+ elif output_format == 'LA':
81
+ output = f"Answer: {lecture} The answer is {answer}."
82
+ elif output_format == 'EA':
83
+ output = f"Answer: {solution} The answer is {answer}."
84
+ elif output_format == 'LEA':
85
+ output = f"Answer: {lecture} {solution} The answer is {answer}."
86
+ elif output_format == 'ELA':
87
+ output = f"Answer: {solution} {lecture} The answer is {answer}."
88
+ elif output_format == 'LEPA':
89
+ output = ''
90
+ if len(lecture.strip()) > 0:
91
+ output += f"LECTURE: {lecture}\n"
92
+ if len(solution.strip()) > 0:
93
+ output += f"SOLUTION: {solution}\n"
94
+ output += '###\n'
95
+ output += f"ANSWER: {answer}."
96
+
97
+ input = input.replace(" ", " ").strip()
98
+ output = output.replace(" ", " ").strip()
99
+ if input.endswith("BECAUSE:"):
100
+ input = input.replace("BECAUSE:", "").strip()
101
+ if output.endswith("BECAUSE:"):
102
+ output = output.replace("BECAUSE:", "").strip()
103
+ return input, output
104
+
105
+
106
+ def create_one_example(format, question, context, choice, answer, lecture, solution, test_example=True):
107
+
108
+ input_format, output_format = format.split("-")
109
+
110
+ ## Inputs
111
+ if input_format == "CQM":
112
+ input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
113
+ elif input_format == "QCM":
114
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
115
+ # upper bound experiment
116
+ elif input_format == "QCML":
117
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
118
+ elif input_format == "QCME":
119
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
120
+ elif input_format == "QCMLE":
121
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
122
+
123
+ elif input_format == "QCLM":
124
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
125
+ elif input_format == "QCEM":
126
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
127
+ elif input_format == "QCLEM":
128
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
129
+
130
+ # Outputs
131
+ if test_example:
132
+ output = "Answer:"
133
+ elif output_format == 'A':
134
+ output = f"Answer: The answer is {answer}."
135
+
136
+ elif output_format == 'AL':
137
+ output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
138
+ elif output_format == 'AE':
139
+ output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
140
+ elif output_format == 'ALE':
141
+ output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
142
+ elif output_format == 'AEL':
143
+ output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
144
+
145
+ elif output_format == 'LA':
146
+ output = f"Answer: {lecture} The answer is {answer}."
147
+ elif output_format == 'EA':
148
+ output = f"Answer: {solution} The answer is {answer}."
149
+ elif output_format == 'LEA':
150
+ output = f"Answer: {lecture} {solution} The answer is {answer}."
151
+ elif output_format == 'ELA':
152
+ output = f"Answer: {solution} {lecture} The answer is {answer}."
153
+
154
+ text = input + output
155
+ text = text.replace(" ", " ").strip()
156
+ if text.endswith("BECAUSE:"):
157
+ text = text.replace("BECAUSE:", "").strip()
158
+ return text
159
+
160
+
161
+
162
+ def create_one_example_gpt4(format, question, context, choice, answer, lecture, solution, test_example=True):
163
+
164
+ input_format, output_format = format.split("-")
165
+
166
+ ## Inputs
167
+ if input_format == "CQM":
168
+ input = f"Context: {context}\nQuestion: {question}\nOptions: {choice}\n"
169
+ elif input_format == "QCM":
170
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\n"
171
+ # upper bound experiment
172
+ elif input_format == "QCML":
173
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture}\n"
174
+ elif input_format == "QCME":
175
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {solution}\n"
176
+ elif input_format == "QCMLE":
177
+ input = f"Question: {question}\nContext: {context}\nOptions: {choice}\nBECAUSE: {lecture} {solution}\n"
178
+
179
+ elif input_format == "QCLM":
180
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture}\nOptions: {choice}\n"
181
+ elif input_format == "QCEM":
182
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {solution}\nOptions: {choice}\n"
183
+ elif input_format == "QCLEM":
184
+ input = f"Question: {question}\nContext: {context}\nBECAUSE: {lecture} {solution}\nOptions: {choice}\n"
185
+
186
+ # Outputs
187
+ if test_example:
188
+ output = "Answer:"
189
+ elif output_format == 'A':
190
+ output = f"Answer: The answer is {answer}."
191
+
192
+ elif output_format == 'AL':
193
+ output = f"Answer: The answer is {answer}. BECAUSE: {solution}"
194
+ elif output_format == 'AE':
195
+ output = f"Answer: The answer is {answer}. BECAUSE: {lecture}"
196
+ elif output_format == 'ALE':
197
+ output = f"Answer: The answer is {answer}. BECAUSE: {lecture} {solution}"
198
+ elif output_format == 'AEL':
199
+ output = f"Answer: The answer is {answer}. BECAUSE: {solution} {lecture}"
200
+
201
+ elif output_format == 'LA':
202
+ output = f"Answer: {lecture} The answer is {answer}."
203
+ elif output_format == 'EA':
204
+ output = f"Answer: {solution} The answer is {answer}."
205
+ elif output_format == 'LEA':
206
+ output = f"Answer: {lecture} {solution} The answer is {answer}."
207
+ elif output_format == 'ELA':
208
+ output = f"Answer: {solution} {lecture} The answer is {answer}."
209
+
210
+ input = input.replace(" ", " ").strip()
211
+ output = output.replace(" ", " ").strip()
212
+ if output.endswith("BECAUSE:"):
213
+ output = output.replace("BECAUSE:", "").strip()
214
+
215
+ user_prompt = {"role": "user", "content": f"Can you explain {input}?"}
216
+ assistant_prompt = {"role": "assistant", "content": f"{output}"}
217
+
218
+ return user_prompt, assistant_prompt
219
+
220
+
221
+ def build_prompt_chatbot(problems, shot_qids, prompt_format, use_caption=False, options=["A", "B", "C", "D", "E"], is_test=False):
222
+ examples = {}
223
+
224
+ for qid in shot_qids:
225
+ question = get_question_text(problems[qid])
226
+ context = get_context_text(problems[qid], use_caption)
227
+ choice = get_choice_text(problems[qid], options)
228
+ answer = get_answer(problems[qid], options)
229
+ lecture = get_lecture_text(problems[qid]).replace('\\n', '\n')
230
+ solution = get_solution_text(problems[qid]).replace('\\n', '\n')
231
+
232
+ train_example = create_one_example_chatbot(prompt_format,
233
+ question,
234
+ context,
235
+ choice,
236
+ answer,
237
+ lecture,
238
+ solution,
239
+ test_example=is_test)
240
+ examples[qid] = train_example
241
+ return examples
242
+
243
+
244
+ def build_prompt(problems, shot_qids, test_qid, args):
245
+
246
+ examples = []
247
+
248
+ # n-shot training examples
249
+ for qid in shot_qids:
250
+ question = get_question_text(problems[qid])
251
+ context = get_context_text(problems[qid], args.use_caption)
252
+ choice = get_choice_text(problems[qid], args.options)
253
+ answer = get_answer(problems[qid], args.options)
254
+ lecture = get_lecture_text(problems[qid])
255
+ solution = get_solution_text(problems[qid])
256
+
257
+ train_example = create_one_example(args.prompt_format,
258
+ question,
259
+ context,
260
+ choice,
261
+ answer,
262
+ lecture,
263
+ solution,
264
+ test_example=False)
265
+ examples.append(train_example)
266
+
267
+ # test example
268
+ question = get_question_text(problems[test_qid])
269
+ context = get_context_text(problems[test_qid], args.use_caption)
270
+ choice = get_choice_text(problems[test_qid], args.options)
271
+ answer = get_answer(problems[test_qid], args.options)
272
+ lecture = get_lecture_text(problems[test_qid])
273
+ solution = get_solution_text(problems[test_qid])
274
+
275
+ test_example = create_one_example(args.prompt_format,
276
+ question,
277
+ context,
278
+ choice,
279
+ answer,
280
+ lecture,
281
+ solution,
282
+ test_example=True)
283
+ examples.append(test_example)
284
+
285
+ # create the prompt input
286
+ prompt_input = '\n\n'.join(examples)
287
+
288
+ return prompt_input
289
+
290
+
291
+ def build_prompt_gpt4(problems, shot_qids, test_qid, args):
292
+
293
+ prompt_array = [{"role": "system", "content": "You are a helpful assistant."}]
294
+
295
+ # n-shot training examples
296
+ for qid in shot_qids:
297
+ question = get_question_text(problems[qid])
298
+ context = get_context_text(problems[qid], args.use_caption)
299
+ choice = get_choice_text(problems[qid], args.options)
300
+ answer = get_answer(problems[qid], args.options)
301
+ lecture = get_lecture_text(problems[qid])
302
+ solution = get_solution_text(problems[qid])
303
+
304
+ user_prompt, assistant_prompt = create_one_example_gpt4(args.prompt_format,
305
+ question,
306
+ context,
307
+ choice,
308
+ answer,
309
+ lecture,
310
+ solution,
311
+ test_example=False)
312
+ prompt_array.append(user_prompt)
313
+ prompt_array.append(assistant_prompt)
314
+
315
+ # test example
316
+ question = get_question_text(problems[test_qid])
317
+ context = get_context_text(problems[test_qid], args.use_caption)
318
+ choice = get_choice_text(problems[test_qid], args.options)
319
+ answer = get_answer(problems[test_qid], args.options)
320
+ lecture = get_lecture_text(problems[test_qid])
321
+ solution = get_solution_text(problems[test_qid])
322
+
323
+ user_prompt, assistant_prompt = create_one_example_gpt4(args.prompt_format,
324
+ question,
325
+ context,
326
+ choice,
327
+ answer,
328
+ lecture,
329
+ solution,
330
+ test_example=True)
331
+ prompt_array.append(user_prompt)
332
+ prompt_array.append(assistant_prompt)
333
+
334
+ return prompt_array
scripts/convert_vizwiz_for_submission.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import argparse
3
+ import json
4
+
5
+ from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
6
+
7
+
8
+ def parse_args():
9
+ parser = argparse.ArgumentParser()
10
+ parser.add_argument('--annotation-file', type=str, required=True)
11
+ parser.add_argument('--result-file', type=str, required=True)
12
+ parser.add_argument('--result-upload-file', type=str, required=True)
13
+ return parser.parse_args()
14
+
15
+
16
+ if __name__ == '__main__':
17
+
18
+ args = parse_args()
19
+
20
+ os.makedirs(os.path.dirname(args.result_upload_file), exist_ok=True)
21
+
22
+ results = []
23
+ error_line = 0
24
+ for line_idx, line in enumerate(open(args.result_file)):
25
+ try:
26
+ results.append(json.loads(line))
27
+ except:
28
+ error_line += 1
29
+ results = {x['question_id']: x['text'] for x in results}
30
+ test_split = [json.loads(line) for line in open(args.annotation_file)]
31
+ split_ids = set([x['question_id'] for x in test_split])
32
+
33
+ print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
34
+
35
+ all_answers = []
36
+
37
+ answer_processor = EvalAIAnswerProcessor()
38
+
39
+ for x in test_split:
40
+ assert x['question_id'] in results
41
+ all_answers.append({
42
+ 'image': x['image'],
43
+ 'answer': answer_processor(results[x['question_id']])
44
+ })
45
+
46
+ with open(args.result_upload_file, 'w') as f:
47
+ json.dump(all_answers, f)
scripts/convert_vqav2_for_submission.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import argparse
3
+ import json
4
+
5
+ from llava.eval.m4c_evaluator import EvalAIAnswerProcessor
6
+
7
+
8
+ def parse_args():
9
+ parser = argparse.ArgumentParser()
10
+ parser.add_argument('--dir', type=str, default="./playground/data/eval/vqav2")
11
+ parser.add_argument('--ckpt', type=str, required=True)
12
+ parser.add_argument('--split', type=str, required=True)
13
+ return parser.parse_args()
14
+
15
+
16
+ if __name__ == '__main__':
17
+
18
+ args = parse_args()
19
+
20
+ src = os.path.join(args.dir, 'answers', args.split, args.ckpt, 'merge.jsonl')
21
+ test_split = os.path.join(args.dir, 'llava_vqav2_mscoco_test2015.jsonl')
22
+ dst = os.path.join(args.dir, 'answers_upload', args.split, f'{args.ckpt}.json')
23
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
24
+
25
+ results = []
26
+ error_line = 0
27
+ for line_idx, line in enumerate(open(src)):
28
+ try:
29
+ results.append(json.loads(line))
30
+ except:
31
+ error_line += 1
32
+
33
+ results = {x['question_id']: x['text'] for x in results}
34
+ test_split = [json.loads(line) for line in open(test_split)]
35
+ split_ids = set([x['question_id'] for x in test_split])
36
+
37
+ print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
38
+
39
+ all_answers = []
40
+
41
+ answer_processor = EvalAIAnswerProcessor()
42
+
43
+ for x in test_split:
44
+ if x['question_id'] not in results:
45
+ all_answers.append({
46
+ 'question_id': x['question_id'],
47
+ 'answer': ''
48
+ })
49
+ else:
50
+ all_answers.append({
51
+ 'question_id': x['question_id'],
52
+ 'answer': answer_processor(results[x['question_id']])
53
+ })
54
+
55
+ with open(dst, 'w') as f:
56
+ json.dump(all_answers, open(dst, 'w'))
scripts/eval_gpt_mmvet.py ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+
3
+ import openai
4
+ import json
5
+ import os
6
+ from tqdm import tqdm
7
+ import pandas as pd
8
+ import numpy as np
9
+ from collections import Counter
10
+ import time
11
+
12
+
13
+
14
+ parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
15
+ parser.add_argument('--mmvet_path')
16
+ parser.add_argument('--ckpt_name')
17
+ parser.add_argument('--result_path')
18
+ args = parser.parse_args()
19
+
20
+
21
+ openai.api_base = "https://api.aiguoguo199.com/v1"
22
+ openai.api_key = 'sk-eionFWpNThMNy4eeFdC25789F60a4cC2A66b2c94D3948bA6'
23
+
24
+ gpt_model = "gpt-3.5-turbo"
25
+
26
+
27
+ prompt = """Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. <AND> in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and <OR> means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score.
28
+
29
+ Question | Ground truth | Prediction | Correctness
30
+ --- | --- | --- | ---
31
+ What is x in the equation? | -1 <AND> -5 | x = 3 | 0.0
32
+ What is x in the equation? | -1 <AND> -5 | x = -1 | 0.5
33
+ What is x in the equation? | -1 <AND> -5 | x = -5 | 0.5
34
+ What is x in the equation? | -1 <AND> -5 | x = -5 or 5 | 0.5
35
+ What is x in the equation? | -1 <AND> -5 | x = -1 or x = -5 | 1.0
36
+ Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.4
37
+ Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.0
38
+ """
39
+
40
+ # load metadata
41
+ # Download mm-vet.zip and `unzip mm-vet.zip` and change the path below
42
+ mmvet_path = args.mmvet_path
43
+ use_sub_set = False
44
+ decimal_places = 1 # number of decimal places to round to
45
+
46
+ if use_sub_set:
47
+ bard_set_file = os.path.join(mmvet_path, "bard_set.json")
48
+ with open(bard_set_file, 'r') as f:
49
+ sub_set = json.load(f)
50
+ sub_set_name = 'bardset'
51
+ sub_set_name = sub_set_name + '_'
52
+ else:
53
+ sub_set = None
54
+ sub_set_name = ''
55
+
56
+ mmvet_metadata = os.path.join(mmvet_path, "mm-vet.json")
57
+ with open(mmvet_metadata, 'r') as f:
58
+ data = json.load(f)
59
+
60
+ counter = Counter()
61
+ cap_set_list = []
62
+ cap_set_counter = []
63
+ len_data = 0
64
+ for id, value in data.items():
65
+ if sub_set is not None and id not in sub_set:
66
+ continue
67
+ question = value["question"]
68
+ answer = value["answer"]
69
+ cap = value["capability"]
70
+ cap = set(cap)
71
+ counter.update(cap)
72
+ if cap not in cap_set_list:
73
+ cap_set_list.append(cap)
74
+ cap_set_counter.append(1)
75
+ else:
76
+ cap_set_counter[cap_set_list.index(cap)] += 1
77
+
78
+ len_data += 1
79
+
80
+ sorted_list = counter.most_common()
81
+ columns = [k for k, v in sorted_list]
82
+ columns.append("total")
83
+ columns.append("std")
84
+ columns.append('runs')
85
+ df = pd.DataFrame(columns=columns)
86
+
87
+ cap_set_sorted_indices = np.argsort(-np.array(cap_set_counter))
88
+ new_cap_set_list = []
89
+ new_cap_set_counter = []
90
+ for index in cap_set_sorted_indices:
91
+ new_cap_set_list.append(cap_set_list[index])
92
+ new_cap_set_counter.append(cap_set_counter[index])
93
+
94
+ cap_set_list = new_cap_set_list
95
+ cap_set_counter = new_cap_set_counter
96
+ cap_set_names = ["_".join(list(cap_set)) for cap_set in cap_set_list]
97
+
98
+ columns2 = cap_set_names
99
+ columns2.append("total")
100
+ columns2.append("std")
101
+ columns2.append('runs')
102
+ df2 = pd.DataFrame(columns=columns2)
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+ ###### change your model name ######
112
+ model = args.ckpt_name
113
+ result_path = args.result_path
114
+ num_run = 1 # we set it as 5 in the paper
115
+ model_results_file = os.path.join(result_path, f"{model}.json")
116
+
117
+ # grade results for each sample to svae
118
+ grade_file = f'{model}_{gpt_model}-grade-{num_run}runs.json'
119
+ grade_file = os.path.join(result_path, grade_file)
120
+
121
+ # score results regarding capabilities/capability integration to save
122
+ cap_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-score-{num_run}runs.csv'
123
+ cap_score_file = os.path.join(result_path, cap_score_file)
124
+ cap_int_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-int-score-{num_run}runs.csv'
125
+ cap_int_score_file = os.path.join(result_path, cap_int_score_file)
126
+
127
+ with open(model_results_file) as f:
128
+ results = json.load(f)
129
+ if os.path.exists(grade_file):
130
+ with open(grade_file, 'r') as f:
131
+ grade_results = json.load(f)
132
+ else:
133
+ grade_results = {}
134
+
135
+
136
+ def need_more_runs():
137
+ need_more_runs = False
138
+ if len(grade_results) > 0:
139
+ for k, v in grade_results.items():
140
+ if len(v['score']) < num_run:
141
+ need_more_runs = True
142
+ break
143
+ return need_more_runs or len(grade_results) < len_data
144
+
145
+
146
+ while need_more_runs():
147
+ for j in range(num_run):
148
+ print(f'eval run {j}')
149
+ for id, line in tqdm(data.items()):
150
+ if sub_set is not None and id not in sub_set:
151
+ continue
152
+ if id in grade_results and len(grade_results[id]['score']) >= (j + 1):
153
+ continue
154
+
155
+ model_pred = results[id]
156
+
157
+ question = prompt + '\n' + ' | '.join(
158
+ [line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "), model_pred,
159
+ ""])
160
+ messages = [
161
+ {"role": "user", "content": question},
162
+ ]
163
+
164
+ if id not in grade_results:
165
+ sample_grade = {'model': [], 'content': [], 'score': []}
166
+ else:
167
+ sample_grade = grade_results[id]
168
+
169
+ grade_sample_run_complete = False
170
+ temperature = 0.0
171
+
172
+ while not grade_sample_run_complete:
173
+ try:
174
+ response = openai.ChatCompletion.create(
175
+ model=gpt_model,
176
+ max_tokens=3,
177
+ temperature=temperature,
178
+ messages=messages)
179
+ # print(response['model'])
180
+ content = response['choices'][0]['message']['content']
181
+ flag = True
182
+ try_time = 1
183
+ while flag:
184
+ try:
185
+ content = content.split(' ')[0].strip()
186
+ score = float(content)
187
+ if score > 1.0 or score < 0.0:
188
+ assert False
189
+ flag = False
190
+ except:
191
+ question = prompt + '\n' + ' | '.join(
192
+ [line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "),
193
+ model_pred, ""]) + "\nPredict the correctness of the answer (digit): "
194
+ messages = [
195
+ {"role": "user", "content": question},
196
+ ]
197
+ response = openai.ChatCompletion.create(
198
+ model=gpt_model,
199
+ max_tokens=3,
200
+ temperature=temperature,
201
+ messages=messages)
202
+ # print(response)
203
+ content = response['choices'][0]['message']['content']
204
+ try_time += 1
205
+ temperature += 0.5
206
+ print(f"{id} try {try_time} times")
207
+ print(content)
208
+ if try_time > 5:
209
+ score = 0.0
210
+ flag = False
211
+ grade_sample_run_complete = True
212
+ except:
213
+ # gpt4 may have token rate limit
214
+ print("sleep 1s")
215
+ time.sleep(1)
216
+
217
+ if len(sample_grade['model']) >= j + 1:
218
+ # sample_grade['model'][j] = response['model']
219
+ sample_grade['content'][j] = content
220
+ sample_grade['score'][j] = score
221
+ else:
222
+ # sample_grade['model'].append(response['model'])
223
+ sample_grade['content'].append(content)
224
+ sample_grade['score'].append(score)
225
+ grade_results[id] = sample_grade
226
+
227
+ with open(grade_file, 'w') as f:
228
+ json.dump(grade_results, f, indent=4)
229
+
230
+ assert not need_more_runs()
231
+ cap_socres = {k: [0.0] * num_run for k in columns[:-2]}
232
+ counter['total'] = len_data
233
+
234
+ cap_socres2 = {k: [0.0] * num_run for k in columns2[:-2]}
235
+ counter2 = {columns2[i]: cap_set_counter[i] for i in range(len(cap_set_counter))}
236
+ counter2['total'] = len_data
237
+
238
+ for k, v in grade_results.items():
239
+ if sub_set is not None and k not in sub_set:
240
+ continue
241
+ for i in range(num_run):
242
+ score = v['score'][i]
243
+ caps = set(data[k]['capability'])
244
+ for c in caps:
245
+ cap_socres[c][i] += score
246
+
247
+ cap_socres['total'][i] += score
248
+
249
+ index = cap_set_list.index(caps)
250
+ cap_socres2[cap_set_names[index]][i] += score
251
+ cap_socres2['total'][i] += score
252
+
253
+ for k, v in cap_socres.items():
254
+ cap_socres[k] = np.array(v) / counter[k] * 100
255
+
256
+ std = round(cap_socres['total'].std(), decimal_places)
257
+ total_copy = cap_socres['total'].copy()
258
+ runs = str(list(np.round(total_copy, decimal_places)))
259
+
260
+ for k, v in cap_socres.items():
261
+ cap_socres[k] = round(v.mean(), decimal_places)
262
+
263
+ cap_socres['std'] = std
264
+ cap_socres['runs'] = runs
265
+ df.loc[model] = cap_socres
266
+
267
+ for k, v in cap_socres2.items():
268
+ cap_socres2[k] = round(np.mean(np.array(v) / counter2[k] * 100), decimal_places)
269
+ cap_socres2['std'] = std
270
+ cap_socres2['runs'] = runs
271
+ df2.loc[model] = cap_socres2
272
+
273
+ df.to_csv(cap_score_file)
274
+ df2.to_csv(cap_int_score_file)
275
+ print(df)
276
+ print(df2)
scripts/finetune.sh ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
4
+
5
+ # Uncomment and set the following variables correspondingly to run this script:
6
+
7
+ ################## VICUNA ##################
8
+ # PROMPT_VERSION=v1
9
+ # MODEL_VERSION="vicuna-v1-3-7b"
10
+ ################## VICUNA ##################
11
+
12
+ ################## LLaMA-2 ##################
13
+ # PROMPT_VERSION="llava_llama_2"
14
+ # MODEL_VERSION="llama-2-7b-chat"
15
+ ################## LLaMA-2 ##################
16
+
17
+ deepspeed llava/train/train_mem.py \
18
+ --deepspeed ./scripts/zero2.json \
19
+ --model_name_or_path ./checkpoints/$MODEL_VERSION \
20
+ --version $PROMPT_VERSION \
21
+ --data_path ./playground/data/llava_instruct_80k.json \
22
+ --image_folder /path/to/coco/train2017 \
23
+ --vision_tower openai/clip-vit-large-patch14 \
24
+ --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
25
+ --mm_vision_select_layer -2 \
26
+ --mm_use_im_start_end False \
27
+ --mm_use_im_patch_token False \
28
+ --bf16 True \
29
+ --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune \
30
+ --num_train_epochs 1 \
31
+ --per_device_train_batch_size 16 \
32
+ --per_device_eval_batch_size 4 \
33
+ --gradient_accumulation_steps 1 \
34
+ --evaluation_strategy "no" \
35
+ --save_strategy "steps" \
36
+ --save_steps 50000 \
37
+ --save_total_limit 1 \
38
+ --learning_rate 2e-5 \
39
+ --weight_decay 0. \
40
+ --warmup_ratio 0.03 \
41
+ --lr_scheduler_type "cosine" \
42
+ --logging_steps 1 \
43
+ --tf32 True \
44
+ --model_max_length 2048 \
45
+ --gradient_checkpointing True \
46
+ --dataloader_num_workers 4 \
47
+ --lazy_preprocess True \
48
+ --report_to wandb
scripts/finetune_full_schedule.sh ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
4
+
5
+ # Uncomment and set the following variables correspondingly to run this script:
6
+
7
+ ################## VICUNA ##################
8
+ # PROMPT_VERSION=v1
9
+ # MODEL_VERSION="vicuna-v1-3-7b"
10
+ ################## VICUNA ##################
11
+
12
+ ################## LLaMA-2 ##################
13
+ # PROMPT_VERSION="llava_llama_2"
14
+ # MODEL_VERSION="llama-2-7b-chat"
15
+ ################## LLaMA-2 ##################
16
+
17
+ deepspeed llava/train/train_mem.py \
18
+ --deepspeed ./scripts/zero2.json \
19
+ --model_name_or_path ./checkpoints/$MODEL_VERSION \
20
+ --version $PROMPT_VERSION \
21
+ --data_path ./playground/data/llava_instruct_158k.json \
22
+ --image_folder /path/to/coco/train2017 \
23
+ --vision_tower openai/clip-vit-large-patch14 \
24
+ --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
25
+ --mm_vision_select_layer -2 \
26
+ --mm_use_im_start_end False \
27
+ --mm_use_im_patch_token False \
28
+ --bf16 True \
29
+ --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune \
30
+ --num_train_epochs 3 \
31
+ --per_device_train_batch_size 16 \
32
+ --per_device_eval_batch_size 4 \
33
+ --gradient_accumulation_steps 1 \
34
+ --evaluation_strategy "no" \
35
+ --save_strategy "steps" \
36
+ --save_steps 50000 \
37
+ --save_total_limit 1 \
38
+ --learning_rate 2e-5 \
39
+ --weight_decay 0. \
40
+ --warmup_ratio 0.03 \
41
+ --lr_scheduler_type "cosine" \
42
+ --logging_steps 1 \
43
+ --tf32 True \
44
+ --model_max_length 2048 \
45
+ --gradient_checkpointing True \
46
+ --dataloader_num_workers 4 \
47
+ --lazy_preprocess True \
48
+ --report_to wandb
scripts/finetune_lora.sh ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
4
+
5
+ # Uncomment and set the following variables correspondingly to run this script:
6
+
7
+ ################## VICUNA ##################
8
+ # PROMPT_VERSION=v1
9
+ # MODEL_VERSION="vicuna-v1-3-7b"
10
+ ################## VICUNA ##################
11
+
12
+ ################## LLaMA-2 ##################
13
+ # PROMPT_VERSION="llava_llama_2"
14
+ # MODEL_VERSION="llama-2-7b-chat"
15
+ ################## LLaMA-2 ##################
16
+
17
+ deepspeed llava/train/train_mem.py \
18
+ --deepspeed ./scripts/zero2.json \
19
+ --lora_enable True \
20
+ --model_name_or_path ./checkpoints/$MODEL_VERSION \
21
+ --version $PROMPT_VERSION \
22
+ --data_path ./playground/data/llava_instruct_80k.json \
23
+ --image_folder /path/to/coco/train2017 \
24
+ --vision_tower openai/clip-vit-large-patch14 \
25
+ --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
26
+ --mm_vision_select_layer -2 \
27
+ --mm_use_im_start_end False \
28
+ --mm_use_im_patch_token False \
29
+ --bf16 True \
30
+ --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
31
+ --num_train_epochs 1 \
32
+ --per_device_train_batch_size 16 \
33
+ --per_device_eval_batch_size 4 \
34
+ --gradient_accumulation_steps 1 \
35
+ --evaluation_strategy "no" \
36
+ --save_strategy "steps" \
37
+ --save_steps 50000 \
38
+ --save_total_limit 1 \
39
+ --learning_rate 2e-5 \
40
+ --weight_decay 0. \
41
+ --warmup_ratio 0.03 \
42
+ --lr_scheduler_type "cosine" \
43
+ --logging_steps 1 \
44
+ --tf32 True \
45
+ --model_max_length 2048 \
46
+ --gradient_checkpointing True \
47
+ --lazy_preprocess True \
48
+ --dataloader_num_workers 4 \
49
+ --report_to wandb
scripts/finetune_qlora.sh ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
4
+
5
+ # Uncomment and set the following variables correspondingly to run this script:
6
+
7
+ ################## VICUNA ##################
8
+ # PROMPT_VERSION=v1
9
+ # MODEL_VERSION="vicuna-v1-3-7b"
10
+ ################## VICUNA ##################
11
+
12
+ ################## LLaMA-2 ##################
13
+ # PROMPT_VERSION="llava_llama_2"
14
+ # MODEL_VERSION="llama-2-7b-chat"
15
+ ################## LLaMA-2 ##################
16
+
17
+ deepspeed llava/train/train_mem.py \
18
+ --deepspeed ./scripts/zero2.json \
19
+ --lora_enable True \
20
+ --bits 4 \
21
+ --model_name_or_path ./checkpoints/$MODEL_VERSION \
22
+ --version $PROMPT_VERSION \
23
+ --data_path ./playground/data/llava_instruct_80k.json \
24
+ --image_folder /path/to/coco/train2017 \
25
+ --vision_tower openai/clip-vit-large-patch14 \
26
+ --pretrain_mm_mlp_adapter ./checkpoints/llava-$MODEL_VERSION-pretrain/mm_projector.bin \
27
+ --mm_vision_select_layer -2 \
28
+ --mm_use_im_start_end False \
29
+ --mm_use_im_patch_token False \
30
+ --bf16 True \
31
+ --output_dir ./checkpoints/llava-$MODEL_VERSION-finetune_lora \
32
+ --num_train_epochs 1 \
33
+ --per_device_train_batch_size 16 \
34
+ --per_device_eval_batch_size 4 \
35
+ --gradient_accumulation_steps 1 \
36
+ --evaluation_strategy "no" \
37
+ --save_strategy "steps" \
38
+ --save_steps 50000 \
39
+ --save_total_limit 1 \
40
+ --learning_rate 2e-5 \
41
+ --weight_decay 0. \
42
+ --warmup_ratio 0.03 \
43
+ --lr_scheduler_type "cosine" \
44
+ --logging_steps 1 \
45
+ --tf32 True \
46
+ --model_max_length 2048 \
47
+ --gradient_checkpointing True \
48
+ --lazy_preprocess True \
49
+ --dataloader_num_workers 4 \
50
+ --report_to wandb
scripts/finetune_sqa.sh ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
4
+
5
+ deepspeed llava/train/train_mem.py \
6
+ --deepspeed ./scripts/zero2.json \
7
+ --model_name_or_path lmsys/vicuna-13b-v1.3 \
8
+ --version $PROMPT_VERSION \
9
+ --data_path /Data/ScienceQA/data/scienceqa/llava_train_QCM-LEA.json \
10
+ --image_folder /Data/ScienceQA/data/scienceqa/images/train \
11
+ --vision_tower openai/clip-vit-large-patch14 \
12
+ --pretrain_mm_mlp_adapter ./checkpoints/huggingface/liuhaotian/llava-pretrain-vicuna-13b-v1.3/mm_projector.bin \
13
+ --mm_vision_select_layer -2 \
14
+ --mm_use_im_start_end False \
15
+ --mm_use_im_patch_token False \
16
+ --bf16 True \
17
+ --output_dir ./checkpoints/llava-vicuna-13b-v1.3-pretrain_lcs558k_plain-ScienceQA_QCM_LEA-12e \
18
+ --num_train_epochs 12 \
19
+ --per_device_train_batch_size 16 \
20
+ --per_device_eval_batch_size 4 \
21
+ --gradient_accumulation_steps 1 \
22
+ --evaluation_strategy "no" \
23
+ --save_strategy "steps" \
24
+ --save_steps 50000 \
25
+ --save_total_limit 1 \
26
+ --learning_rate 2e-5 \
27
+ --weight_decay 0. \
28
+ --warmup_ratio 0.03 \
29
+ --lr_scheduler_type "cosine" \
30
+ --logging_steps 1 \
31
+ --tf32 True \
32
+ --model_max_length 2048 \
33
+ --gradient_checkpointing True \
34
+ --dataloader_num_workers 4 \
35
+ --lazy_preprocess True \
36
+ --report_to wandb
scripts/merge_lora_weights.py ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ from llava.model.builder import load_pretrained_model
3
+ from llava.mm_utils import get_model_name_from_path
4
+
5
+
6
+ def merge_lora(args):
7
+ model_name = get_model_name_from_path(args.model_path)
8
+ tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, device_map='cpu')
9
+
10
+ model.save_pretrained(args.save_model_path)
11
+ tokenizer.save_pretrained(args.save_model_path)
12
+
13
+
14
+ if __name__ == "__main__":
15
+ parser = argparse.ArgumentParser()
16
+ parser.add_argument("--model-path", type=str, required=True)
17
+ parser.add_argument("--model-base", type=str, required=True)
18
+ parser.add_argument("--save-model-path", type=str, required=True)
19
+
20
+ args = parser.parse_args()
21
+
22
+ merge_lora(args)
scripts/pretrain.sh ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # IMPORTANT: this is the training script for the original LLaVA, NOT FOR LLaVA V1.5!
4
+
5
+ # Uncomment and set the following variables correspondingly to run this script:
6
+
7
+ # MODEL_VERSION=vicuna-v1-3-7b
8
+ # MODEL_VERSION=llama-2-7b-chat
9
+
10
+ ########### DO NOT CHANGE ###########
11
+ ########### USE THIS FOR BOTH ###########
12
+ PROMPT_VERSION=plain
13
+ ########### DO NOT CHANGE ###########
14
+
15
+ deepspeed llava/train/train_mem.py \
16
+ --deepspeed ./scripts/zero2.json \
17
+ --model_name_or_path ./checkpoints/$MODEL_VERSION \
18
+ --version $PROMPT_VERSION \
19
+ --data_path /path/to/pretrain_data.json \
20
+ --image_folder /path/to/images \
21
+ --vision_tower openai/clip-vit-large-patch14 \
22
+ --tune_mm_mlp_adapter True \
23
+ --mm_vision_select_layer -2 \
24
+ --mm_use_im_start_end False \
25
+ --mm_use_im_patch_token False \
26
+ --bf16 True \
27
+ --output_dir ./checkpoints/llava-$MODEL_VERSION-pretrain \
28
+ --num_train_epochs 1 \
29
+ --per_device_train_batch_size 16 \
30
+ --per_device_eval_batch_size 4 \
31
+ --gradient_accumulation_steps 1 \
32
+ --evaluation_strategy "no" \
33
+ --save_strategy "steps" \
34
+ --save_steps 24000 \
35
+ --save_total_limit 1 \
36
+ --learning_rate 2e-3 \
37
+ --weight_decay 0. \
38
+ --warmup_ratio 0.03 \
39
+ --lr_scheduler_type "cosine" \
40
+ --logging_steps 1 \
41
+ --tf32 True \
42
+ --model_max_length 2048 \
43
+ --gradient_checkpointing True \
44
+ --dataloader_num_workers 4 \
45
+ --lazy_preprocess True \
46
+ --report_to wandb
scripts/sqa_eval_batch.sh ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ CHUNKS=8
4
+ for IDX in {0..7}; do
5
+ CUDA_VISIBLE_DEVICES=$IDX python -m llava.eval.model_vqa_science \
6
+ --model-path liuhaotian/llava-lcs558k-scienceqa-vicuna-13b-v1.3 \
7
+ --question-file ~/haotian/datasets/ScienceQA/data/scienceqa/llava_test_QCM-LEA.json \
8
+ --image-folder ~/haotian/datasets/ScienceQA/data/scienceqa/images/test \
9
+ --answers-file ./test_llava-13b-chunk$CHUNKS_$IDX.jsonl \
10
+ --num-chunks $CHUNKS \
11
+ --chunk-idx $IDX \
12
+ --conv-mode llava_v1 &
13
+ done
scripts/sqa_eval_gather.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ CHUNKS=8
4
+ output_file="test_llava-13b.jsonl"
5
+
6
+ # Clear out the output file if it exists.
7
+ > "$output_file"
8
+
9
+ # Loop through the indices and concatenate each file.
10
+ for idx in $(seq 0 $((CHUNKS-1))); do
11
+ cat "./test_llava-13b-chunk${idx}.jsonl" >> "$output_file"
12
+ done
13
+
14
+ python llava/eval/eval_science_qa.py \
15
+ --base-dir ~/haotian/datasets/ScienceQA/data/scienceqa \
16
+ --result-file ./test_llava-13b.jsonl \
17
+ --output-file ./test_llava-13b_output.json \
18
+ --output-result ./test_llava-13b_result.json
scripts/v1_5/eval/eval_benchmark_1_correctness.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ CKPT_NAME="checkpoints/Video-LLaVA-7B"
3
+ Video_5_Benchmark="eval/Video_5_Benchmark"
4
+ pred_path="${Video_5_Benchmark}/${CKPT_NAME}/correctness_qa.json"
5
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}/gpt3/correctness"
6
+ output_json="${Video_5_Benchmark}/${CKPT_NAME}/results/correctness_qa.json"
7
+ api_key=""
8
+ api_base=""
9
+ num_tasks=8
10
+
11
+ python3 llava/eval/video/eval_benchmark_1_correctness.py \
12
+ --pred_path ${pred_path} \
13
+ --output_dir ${output_dir} \
14
+ --output_json ${output_json} \
15
+ --api_key ${api_key} \
16
+ --api_base ${api_base} \
17
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_benchmark_2_detail.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="checkpoints/Video-LLaVA-7B"
4
+ Video_5_Benchmark="eval/Video_5_Benchmark"
5
+ pred_path="${Video_5_Benchmark}/${CKPT_NAME}/detail_qa.json"
6
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}/gpt3/detail"
7
+ output_json="${Video_5_Benchmark}/${CKPT_NAME}/results/detail_qa.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+ python3 llava/eval/video/eval_benchmark_2_detailed_orientation.py \
13
+ --pred_path ${pred_path} \
14
+ --output_dir ${output_dir} \
15
+ --output_json ${output_json} \
16
+ --api_key ${api_key} \
17
+ --api_base ${api_base} \
18
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_benchmark_3_contextual.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="checkpoints/Video-LLaVA-7B"
4
+ Video_5_Benchmark="eval/Video_5_Benchmark"
5
+ pred_path="${Video_5_Benchmark}/${CKPT_NAME}/contextual_qa.json"
6
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}/gpt3/contextual"
7
+ output_json="${Video_5_Benchmark}/${CKPT_NAME}/results/contextual_qa.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+ python3 llava/eval/video/eval_benchmark_3_context.py \
13
+ --pred_path ${pred_path} \
14
+ --output_dir ${output_dir} \
15
+ --output_json ${output_json} \
16
+ --api_key ${api_key} \
17
+ --api_base ${api_base} \
18
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_benchmark_4_temporal.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="checkpoints/Video-LLaVA-7B"
4
+ Video_5_Benchmark="eval/Video_5_Benchmark"
5
+ pred_path="${Video_5_Benchmark}/${CKPT_NAME}/temporal_qa.json"
6
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}/gpt3/temporal"
7
+ output_json="${Video_5_Benchmark}/${CKPT_NAME}/results/temporal_qa.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+ python3 llava/eval/video/eval_benchmark_4_temporal.py \
13
+ --pred_path ${pred_path} \
14
+ --output_dir ${output_dir} \
15
+ --output_json ${output_json} \
16
+ --api_key ${api_key} \
17
+ --api_base ${api_base} \
18
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_benchmark_5_consistency.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="checkpoints/Video-LLaVA-7B"
4
+ Video_5_Benchmark="eval/Video_5_Benchmark"
5
+ pred_path="${Video_5_Benchmark}/${CKPT_NAME}/consistency_qa.json"
6
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}/gpt3/consistency"
7
+ output_json="${Video_5_Benchmark}/${CKPT_NAME}/results/consistency_qa.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+ python3 llava/eval/video/eval_benchmark_5_consistency.py \
13
+ --pred_path ${pred_path} \
14
+ --output_dir ${output_dir} \
15
+ --output_json ${output_json} \
16
+ --api_key ${api_key} \
17
+ --api_base ${api_base} \
18
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_image_gqa.sh ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
4
+ IFS=',' read -ra GPULIST <<< "$gpu_list"
5
+
6
+ CHUNKS=${#GPULIST[@]}
7
+
8
+ CKPT_NAME="Video-LLaVA-7B"
9
+ CKPT="checkpoints/${CKPT_NAME}"
10
+ SPLIT="llava_gqa_testdev_balanced"
11
+ EVAL="eval"
12
+ GQADIR="${EVAL}/gqa/data"
13
+
14
+ for IDX in $(seq 0 $((CHUNKS-1))); do
15
+ CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python3 -m llava.eval.model_vqa_loader \
16
+ --model-path ${CKPT} \
17
+ --question-file ${EVAL}/gqa/$SPLIT.jsonl \
18
+ --image-folder ${EVAL}/gqa/data/images \
19
+ --answers-file ${EVAL}/gqa/answers/$SPLIT/${CKPT_NAME}/${CHUNKS}_${IDX}.jsonl \
20
+ --num-chunks $CHUNKS \
21
+ --chunk-idx $IDX \
22
+ --temperature 0 \
23
+ --conv-mode vicuna_v1 &
24
+ done
25
+
26
+ wait
27
+
28
+ output_file=${EVAL}/gqa/answers/$SPLIT/${CKPT_NAME}/merge.jsonl
29
+
30
+ # Clear out the output file if it exists.
31
+ > "$output_file"
32
+
33
+ # Loop through the indices and concatenate each file.
34
+ for IDX in $(seq 0 $((CHUNKS-1))); do
35
+ cat ${EVAL}/gqa/answers/$SPLIT/${CKPT_NAME}/${CHUNKS}_${IDX}.jsonl >> "$output_file"
36
+ done
37
+
38
+ mkdir -p $GQADIR/$SPLIT/${CKPT_NAME}
39
+ python3 scripts/convert_gqa_for_eval.py --src $output_file --dst $GQADIR/$SPLIT/${CKPT_NAME}/testdev_balanced_predictions.json
40
+
41
+ cd $GQADIR
42
+ python3 eval/eval_gqa.py --tier $SPLIT/${CKPT_NAME}/testdev_balanced \
43
+ --questions /scc_cephfs/yy/lb/LLaVA-Video-YY/questions1.2/testdev_balanced_questions.json
scripts/v1_5/eval/eval_image_llavabench.sh ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ CKPT="checkpoints/${CKPT_NAME}"
5
+ EVAL="eval"
6
+ python3 -m llava.eval.model_vqa \
7
+ --model-path ${CKPT} \
8
+ --question-file ${EVAL}/llava-bench-in-the-wild/questions.jsonl \
9
+ --image-folder ${EVAL}/llava-bench-in-the-wild/images \
10
+ --answers-file ${EVAL}/llava-bench-in-the-wild/answers/${CKPT_NAME}.jsonl \
11
+ --temperature 0 \
12
+ --conv-mode vicuna_v1
13
+
14
+ mkdir -p ${EVAL}/llava-bench-in-the-wild/reviews
15
+
16
+ python3 llava/eval/eval_gpt_review_bench.py \
17
+ --question ${EVAL}/llava-bench-in-the-wild/questions.jsonl \
18
+ --context ${EVAL}/llava-bench-in-the-wild/context.jsonl \
19
+ --rule llava/eval/table/rule.json \
20
+ --answer-list ${EVAL}/llava-bench-in-the-wild/answers_gpt4.jsonl \
21
+ ${EVAL}/llava-bench-in-the-wild/answers/${CKPT_NAME}.jsonl \
22
+ --output ${EVAL}/llava-bench-in-the-wild/reviews/${CKPT_NAME}.jsonl
23
+
24
+ python3 llava/eval/summarize_gpt_review.py -f ${EVAL}/llava-bench-in-the-wild/reviews/${CKPT_NAME}.jsonl
scripts/v1_5/eval/eval_image_mmbench.sh ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ SPLIT="mmbench_dev_20230712"
4
+
5
+ CKPT_NAME="Video-LLaVA-7B"
6
+ CKPT="checkpoints/${CKPT_NAME}"
7
+ EVAL="eval"
8
+ python3 -m llava.eval.model_vqa_mmbench \
9
+ --model-path ${CKPT} \
10
+ --question-file ${EVAL}/mmbench/$SPLIT.tsv \
11
+ --answers-file ${EVAL}/mmbench/answers/$SPLIT/${CKPT_NAME}.jsonl \
12
+ --single-pred-prompt \
13
+ --temperature 0 \
14
+ --conv-mode vicuna_v1
15
+
16
+ mkdir -p ${EVAL}/mmbench/answers_upload/$SPLIT
17
+
18
+ python3 scripts/convert_mmbench_for_submission.py \
19
+ --annotation-file ${EVAL}/mmbench/$SPLIT.tsv \
20
+ --result-dir ${EVAL}/mmbench/answers/$SPLIT \
21
+ --upload-dir ${EVAL}/mmbench/answers_upload/$SPLIT \
22
+ --experiment ${CKPT_NAME}
scripts/v1_5/eval/eval_image_mmvet.sh ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ CKPT="checkpoints/${CKPT_NAME}"
5
+ EVAL="eval"
6
+ python3 -m llava.eval.model_vqa \
7
+ --model-path ${CKPT} \
8
+ --question-file ${EVAL}/mm-vet/llava-mm-vet.jsonl \
9
+ --image-folder ${EVAL}/mm-vet/images \
10
+ --answers-file ${EVAL}/mm-vet/answers/${CKPT_NAME}.jsonl \
11
+ --temperature 0 \
12
+ --conv-mode vicuna_v1
13
+
14
+ mkdir -p ${EVAL}/mm-vet/results
15
+
16
+ python3 scripts/convert_mmvet_for_eval.py \
17
+ --src ${EVAL}/mm-vet/answers/${CKPT_NAME}.jsonl \
18
+ --dst ${EVAL}/mm-vet/results/${CKPT_NAME}.json
19
+
20
+
21
+ python3 scripts/eval_gpt_mmvet.py \
22
+ --mmvet_path ${EVAL}/mm-vet \
23
+ --ckpt_name ${CKPT_NAME} \
24
+ --result_path ${EVAL}/mm-vet/results
scripts/v1_5/eval/eval_image_pope.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+
4
+ CKPT_NAME="Video-LLaVA-7B"
5
+ CKPT="checkpoints/${CKPT_NAME}"
6
+ EVAL="eval"
7
+ python3 -m llava.eval.model_vqa_loader \
8
+ --model-path ${CKPT} \
9
+ --question-file ${EVAL}/pope/llava_pope_test.jsonl \
10
+ --image-folder ${EVAL}/pope/val2014 \
11
+ --answers-file ${EVAL}/pope/answers/${CKPT_NAME}.jsonl \
12
+ --temperature 0 \
13
+ --conv-mode vicuna_v1
14
+
15
+ python3 llava/eval/eval_pope.py \
16
+ --annotation-dir ${EVAL}/pope/coco \
17
+ --question-file ${EVAL}/pope/llava_pope_test.jsonl \
18
+ --result-file ${EVAL}/pope/answers/${CKPT_NAME}.jsonl
scripts/v1_5/eval/eval_image_sqa.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+
4
+ CKPT_NAME="Video-LLaVA-7B"
5
+ CKPT="checkpoints/${CKPT_NAME}"
6
+ EVAL="eval"
7
+ python3 -m llava.eval.model_vqa_science \
8
+ --model-path ${CKPT} \
9
+ --question-file ${EVAL}/scienceqa/llava_test_CQM-A.json \
10
+ --image-folder ${EVAL}/scienceqa/images/test \
11
+ --answers-file ${EVAL}/scienceqa/answers/${CKPT_NAME}.jsonl \
12
+ --single-pred-prompt \
13
+ --temperature 0 \
14
+ --conv-mode vicuna_v1
15
+
16
+ python3 llava/eval/eval_science_qa.py \
17
+ --base-dir ${EVAL}/scienceqa \
18
+ --result-file ${EVAL}/scienceqa/answers/${CKPT_NAME}.jsonl \
19
+ --output-file ${EVAL}/scienceqa/answers/${CKPT_NAME}_output.jsonl \
20
+ --output-result ${EVAL}/scienceqa/answers/${CKPT_NAME}_result.json
scripts/v1_5/eval/eval_image_textvqa.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+
4
+ CKPT_NAME="Video-LLaVA-7B"
5
+ CKPT="checkpoints/${CKPT_NAME}"
6
+ EVAL="eval"
7
+ python3 -m llava.eval.model_vqa_loader \
8
+ --model-path ${CKPT} \
9
+ --question-file ${EVAL}/textvqa/llava_textvqa_val_v051_ocr.jsonl \
10
+ --image-folder ${EVAL}/textvqa/train_images \
11
+ --answers-file ${EVAL}/textvqa/answers/${CKPT_NAME}.jsonl \
12
+ --temperature 0 \
13
+ --conv-mode vicuna_v1
14
+
15
+ python3 -m llava.eval.eval_textvqa \
16
+ --annotation-file ${EVAL}/textvqa/TextVQA_0.5.1_val.json \
17
+ --result-file ${EVAL}/textvqa/answers/${CKPT_NAME}.jsonl
scripts/v1_5/eval/eval_image_vizwiz.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ CKPT="checkpoints/${CKPT_NAME}"
5
+ EVAL="eval"
6
+ python3 -m llava.eval.model_vqa_loader \
7
+ --model-path ${CKPT} \
8
+ --question-file ${EVAL}/vizwiz/llava_test.jsonl \
9
+ --image-folder ${EVAL}/vizwiz/test \
10
+ --answers-file ${EVAL}/vizwiz/answers/${CKPT_NAME}.jsonl \
11
+ --temperature 0 \
12
+ --conv-mode vicuna_v1
13
+
14
+ python3 scripts/convert_vizwiz_for_submission.py \
15
+ --annotation-file ${EVAL}/vizwiz/llava_test.jsonl \
16
+ --result-file ${EVAL}/vizwiz/answers/${CKPT_NAME}.jsonl \
17
+ --result-upload-file ${EVAL}/vizwiz/answers_upload/${CKPT_NAME}.json
scripts/v1_5/eval/eval_image_vqav2.sh ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
4
+ IFS=',' read -ra GPULIST <<< "$gpu_list"
5
+
6
+ CHUNKS=${#GPULIST[@]}
7
+
8
+ CKPT_NAME="Video-LLaVA-7B"
9
+ CKPT="checkpoints/${CKPT_NAME}"
10
+ SPLIT="llava_vqav2_mscoco_test-dev2015"
11
+ EVAL="eval"
12
+
13
+ for IDX in $(seq 0 $((CHUNKS-1))); do
14
+ CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python3 -m llava.eval.model_vqa_loader \
15
+ --model-path ${CKPT} \
16
+ --question-file ${EVAL}/vqav2/$SPLIT.jsonl \
17
+ --image-folder ${EVAL}/vqav2/test2015 \
18
+ --answers-file ${EVAL}/vqav2/answers/$SPLIT/${CKPT_NAME}/${CHUNKS}_${IDX}.jsonl \
19
+ --num-chunks $CHUNKS \
20
+ --chunk-idx $IDX \
21
+ --temperature 0 \
22
+ --conv-mode vicuna_v1 &
23
+ done
24
+
25
+ wait
26
+
27
+ output_file=${EVAL}/vqav2/answers/$SPLIT/${CKPT_NAME}/merge.jsonl
28
+
29
+ # Clear out the output file if it exists.
30
+ > "$output_file"
31
+
32
+ # Loop through the indices and concatenate each file.
33
+ for IDX in $(seq 0 $((CHUNKS-1))); do
34
+ cat ${EVAL}/vqav2/answers/$SPLIT/${CKPT_NAME}/${CHUNKS}_${IDX}.jsonl >> "$output_file"
35
+ done
36
+
37
+ python3 scripts/convert_vqav2_for_submission.py --split $SPLIT --ckpt ${CKPT_NAME} --dir ${EVAL}/vqav2
38
+
scripts/v1_5/eval/eval_qa_activitynet.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
4
+ output_name="Video-LLaVA-7B"
5
+ pred_path="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/${output_name}/merge.jsonl"
6
+ output_dir="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/${output_name}/gpt3-0.25"
7
+ output_json="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/${output_name}/results.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+
13
+
14
+ python3 llava/eval/video/eval_video_qa.py \
15
+ --pred_path ${pred_path} \
16
+ --output_dir ${output_dir} \
17
+ --output_json ${output_json} \
18
+ --api_key ${api_key} \
19
+ --api_base ${api_base} \
20
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_qa_msrvtt.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
4
+ output_name="Video-LLaVA-7B"
5
+ pred_path="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/${output_name}/merge.jsonl"
6
+ output_dir="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/${output_name}/gpt"
7
+ output_json="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/${output_name}/results.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+
13
+
14
+ python3 llava/eval/video/eval_video_qa.py \
15
+ --pred_path ${pred_path} \
16
+ --output_dir ${output_dir} \
17
+ --output_json ${output_json} \
18
+ --api_key ${api_key} \
19
+ --api_base ${api_base} \
20
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_qa_msvd.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
4
+ output_name="Video-LLaVA-7B"
5
+ pred_path="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/${output_name}/merge.jsonl"
6
+ output_dir="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/${output_name}/gpt3.5"
7
+ output_json="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/${output_name}/results.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+
13
+
14
+ python3 llava/eval/video/eval_video_qa.py \
15
+ --pred_path ${pred_path} \
16
+ --output_dir ${output_dir} \
17
+ --output_json ${output_json} \
18
+ --api_key ${api_key} \
19
+ --api_base ${api_base} \
20
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/eval_qa_tgif.sh ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
4
+ output_name="Video-LLaVA-7B"
5
+ pred_path="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/${output_name}/merge.jsonl"
6
+ output_dir="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/${output_name}/gpt3.5-0.0"
7
+ output_json="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/${output_name}/results.json"
8
+ api_key=""
9
+ api_base=""
10
+ num_tasks=8
11
+
12
+
13
+
14
+ python3 llava/eval/video/eval_video_qa.py \
15
+ --pred_path ${pred_path} \
16
+ --output_dir ${output_dir} \
17
+ --output_json ${output_json} \
18
+ --api_key ${api_key} \
19
+ --api_base ${api_base} \
20
+ --num_tasks ${num_tasks}
scripts/v1_5/eval/run_benchmark_1_correctness.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ Video_5_Benchmark="eval/Video_5_Benchmark"
7
+ video_dir="${Video_5_Benchmark}/Test_Videos"
8
+ gt_file="${Video_5_Benchmark}/Benchmarking_QA/generic_qa.json"
9
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}"
10
+ output_name="correctness_qa"
11
+
12
+ python3 llava/eval/video/run_inference_benchmark_general.py \
13
+ --model_path ${model_path} \
14
+ --cache_dir ${cache_dir} \
15
+ --video_dir ${video_dir} \
16
+ --gt_file ${gt_file} \
17
+ --output_dir ${output_dir} \
18
+ --output_name ${output_name}
scripts/v1_5/eval/run_benchmark_2_detail.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ Video_5_Benchmark="eval/Video_5_Benchmark"
7
+ video_dir="${Video_5_Benchmark}/Test_Videos"
8
+ gt_file="${Video_5_Benchmark}/Benchmarking_QA/generic_qa.json"
9
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}"
10
+ output_name="detail_qa"
11
+
12
+ python3 llava/eval/video/run_inference_benchmark_general.py \
13
+ --model_path ${model_path} \
14
+ --cache_dir ${cache_dir} \
15
+ --video_dir ${video_dir} \
16
+ --gt_file ${gt_file} \
17
+ --output_dir ${output_dir} \
18
+ --output_name ${output_name}
scripts/v1_5/eval/run_benchmark_3_contextual.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ Video_5_Benchmark="eval/Video_5_Benchmark"
7
+ video_dir="${Video_5_Benchmark}/Test_Videos"
8
+ gt_file="${Video_5_Benchmark}/Benchmarking_QA/generic_qa.json"
9
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}"
10
+ output_name="contextual_qa"
11
+
12
+ python3 llava/eval/video/run_inference_benchmark_general.py \
13
+ --model_path ${model_path} \
14
+ --cache_dir ${cache_dir} \
15
+ --video_dir ${video_dir} \
16
+ --gt_file ${gt_file} \
17
+ --output_dir ${output_dir} \
18
+ --output_name ${output_name}
scripts/v1_5/eval/run_benchmark_4_temporal.sh ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ CKPT_NAME="Video-LLaVA-7B"
3
+ model_path="checkpoints/${CKPT_NAME}"
4
+ cache_dir="./cache_dir"
5
+ Video_5_Benchmark="eval/Video_5_Benchmark"
6
+ video_dir="${Video_5_Benchmark}/Test_Videos"
7
+ gt_file="${Video_5_Benchmark}/Benchmarking_QA/temporal_qa.json"
8
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}"
9
+ output_name="temporal_qa"
10
+
11
+ python3 llava/eval/video/run_inference_benchmark_general.py \
12
+ --model_path ${model_path} \
13
+ --cache_dir ${cache_dir} \
14
+ --video_dir ${video_dir} \
15
+ --gt_file ${gt_file} \
16
+ --output_dir ${output_dir} \
17
+ --output_name ${output_name}
scripts/v1_5/eval/run_benchmark_5_consistency.sh ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ Video_5_Benchmark="eval/Video_5_Benchmark"
7
+ video_dir="${Video_5_Benchmark}/Test_Videos"
8
+ gt_file="${Video_5_Benchmark}/Benchmarking_QA/consistency_qa.json"
9
+ output_dir="${Video_5_Benchmark}/${CKPT_NAME}"
10
+ output_name="consistency_qa"
11
+
12
+ python3 llava/eval/video/run_inference_benchmark_consistency.py \
13
+ --model_path ${model_path} \
14
+ --cache_dir ${cache_dir} \
15
+ --video_dir ${video_dir} \
16
+ --gt_file ${gt_file} \
17
+ --output_dir ${output_dir} \
18
+ --output_name ${output_name}
scripts/v1_5/eval/run_qa_activitynet.sh ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
7
+ video_dir="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/all_test"
8
+ gt_file_question="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/test_q.json"
9
+ gt_file_answers="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/test_a.json"
10
+ output_dir="${GPT_Zero_Shot_QA}/Activitynet_Zero_Shot_QA/${CKPT_NAME}"
11
+
12
+
13
+ gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
14
+ IFS=',' read -ra GPULIST <<< "$gpu_list"
15
+
16
+ CHUNKS=${#GPULIST[@]}
17
+
18
+
19
+ for IDX in $(seq 0 $((CHUNKS-1))); do
20
+ CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python3 llava/eval/video/run_inference_video_qa_act.py \
21
+ --model_path ${model_path} \
22
+ --cache_dir ${cache_dir} \
23
+ --video_dir ${video_dir} \
24
+ --gt_file_question ${gt_file_question} \
25
+ --gt_file_answers ${gt_file_answers} \
26
+ --output_dir ${output_dir} \
27
+ --output_name ${CHUNKS}_${IDX} \
28
+ --num_chunks $CHUNKS \
29
+ --chunk_idx $IDX &
30
+ done
31
+
32
+ wait
33
+
34
+ output_file=${output_dir}/merge.jsonl
35
+
36
+ # Clear out the output file if it exists.
37
+ > "$output_file"
38
+
39
+ # Loop through the indices and concatenate each file.
40
+ for IDX in $(seq 0 $((CHUNKS-1))); do
41
+ cat ${output_dir}/${CHUNKS}_${IDX}.json >> "$output_file"
42
+ done
scripts/v1_5/eval/run_qa_msrvtt.sh ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
7
+ video_dir="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/videos/all"
8
+ gt_file_question="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/test_q.json"
9
+ gt_file_answers="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/test_a.json"
10
+ output_dir="${GPT_Zero_Shot_QA}/MSRVTT_Zero_Shot_QA/${CKPT_NAME}"
11
+
12
+
13
+
14
+ gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
15
+ IFS=',' read -ra GPULIST <<< "$gpu_list"
16
+
17
+ CHUNKS=${#GPULIST[@]}
18
+
19
+
20
+ for IDX in $(seq 0 $((CHUNKS-1))); do
21
+ CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python3 llava/eval/video/run_inference_video_qa.py \
22
+ --model_path ${model_path} \
23
+ --cache_dir ${cache_dir} \
24
+ --video_dir ${video_dir} \
25
+ --gt_file_question ${gt_file_question} \
26
+ --gt_file_answers ${gt_file_answers} \
27
+ --output_dir ${output_dir} \
28
+ --output_name ${CHUNKS}_${IDX} \
29
+ --num_chunks $CHUNKS \
30
+ --chunk_idx $IDX &
31
+ done
32
+
33
+ wait
34
+
35
+ output_file=${output_dir}/merge.jsonl
36
+
37
+ # Clear out the output file if it exists.
38
+ > "$output_file"
39
+
40
+ # Loop through the indices and concatenate each file.
41
+ for IDX in $(seq 0 $((CHUNKS-1))); do
42
+ cat ${output_dir}/${CHUNKS}_${IDX}.json >> "$output_file"
43
+ done
scripts/v1_5/eval/run_qa_msvd.sh ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
7
+ video_dir="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/videos"
8
+ gt_file_question="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/test_q.json"
9
+ gt_file_answers="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/test_a.json"
10
+ output_dir="${GPT_Zero_Shot_QA}/MSVD_Zero_Shot_QA/${CKPT_NAME}"
11
+
12
+
13
+ gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
14
+ IFS=',' read -ra GPULIST <<< "$gpu_list"
15
+
16
+ CHUNKS=${#GPULIST[@]}
17
+
18
+
19
+ for IDX in $(seq 0 $((CHUNKS-1))); do
20
+ CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python3 llava/eval/video/run_inference_video_qa.py \
21
+ --model_path ${model_path} \
22
+ --cache_dir ${cache_dir} \
23
+ --video_dir ${video_dir} \
24
+ --gt_file_question ${gt_file_question} \
25
+ --gt_file_answers ${gt_file_answers} \
26
+ --output_dir ${output_dir} \
27
+ --output_name ${CHUNKS}_${IDX} \
28
+ --num_chunks $CHUNKS \
29
+ --chunk_idx $IDX &
30
+ done
31
+
32
+ wait
33
+
34
+ output_file=${output_dir}/merge.jsonl
35
+
36
+ # Clear out the output file if it exists.
37
+ > "$output_file"
38
+
39
+ # Loop through the indices and concatenate each file.
40
+ for IDX in $(seq 0 $((CHUNKS-1))); do
41
+ cat ${output_dir}/${CHUNKS}_${IDX}.json >> "$output_file"
42
+ done
scripts/v1_5/eval/run_qa_tgif.sh ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ CKPT_NAME="Video-LLaVA-7B"
4
+ model_path="checkpoints/${CKPT_NAME}"
5
+ cache_dir="./cache_dir"
6
+ GPT_Zero_Shot_QA="eval/GPT_Zero_Shot_QA"
7
+ video_dir="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/mp4"
8
+ gt_file_question="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/test_q.json"
9
+ gt_file_answers="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/test_a.json"
10
+ output_dir="${GPT_Zero_Shot_QA}/TGIF_Zero_Shot_QA/${CKPT_NAME}"
11
+
12
+
13
+ gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
14
+ IFS=',' read -ra GPULIST <<< "$gpu_list"
15
+
16
+ CHUNKS=${#GPULIST[@]}
17
+
18
+
19
+ for IDX in $(seq 0 $((CHUNKS-1))); do
20
+ CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python3 llava/eval/video/run_inference_video_qa.py \
21
+ --model_path ${model_path} \
22
+ --cache_dir ${cache_dir} \
23
+ --video_dir ${video_dir} \
24
+ --gt_file_question ${gt_file_question} \
25
+ --gt_file_answers ${gt_file_answers} \
26
+ --output_dir ${output_dir} \
27
+ --output_name ${CHUNKS}_${IDX} \
28
+ --num_chunks $CHUNKS \
29
+ --chunk_idx $IDX &
30
+ done
31
+
32
+ wait
33
+
34
+ output_file=${output_dir}/merge.jsonl
35
+
36
+ # Clear out the output file if it exists.
37
+ > "$output_file"
38
+
39
+ # Loop through the indices and concatenate each file.
40
+ for IDX in $(seq 0 $((CHUNKS-1))); do
41
+ cat ${output_dir}/${CHUNKS}_${IDX}.json >> "$output_file"
42
+ done
scripts/v1_5/finetune.sh ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ DATA_ROOT="llava_all_image_video"
4
+ HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 CUDA_VISIBLE_DEVICES=0,1,2,3 deepspeed llava/train/train_mem.py \
5
+ --deepspeed ./scripts/zero2.json \
6
+ --model_name_or_path lmsys/vicuna-7b-v1.5 \
7
+ --version v1 \
8
+ --data_path train_json/videochatgpt_llavaimage_tune.json \
9
+ --video_folder ${DATA_ROOT} \
10
+ --image_folder ${DATA_ROOT} \
11
+ --X "Video" "Image" \
12
+ --video_tower LanguageBind/LanguageBind_Video \
13
+ --image_tower LanguageBind/LanguageBind_Image \
14
+ --pretrain_mm_mlp_adapter checkpoints/Video-LLaVA-Pretrain-7B/mm_projector.bin \
15
+ --mm_projector_type mlp2x_gelu \
16
+ --mm_vision_select_layer -2 \
17
+ --mm_use_x_start_end False \
18
+ --mm_use_x_patch_token False \
19
+ --image_aspect_ratio pad \
20
+ --group_by_modality_length True \
21
+ --bf16 True \
22
+ --output_dir ./checkpoints/Video-LLaVA-7B \
23
+ --num_train_epochs 1 \
24
+ --per_device_train_batch_size 16 \
25
+ --per_device_eval_batch_size 4 \
26
+ --gradient_accumulation_steps 2 \
27
+ --evaluation_strategy "no" \
28
+ --save_strategy "steps" \
29
+ --save_steps 50000 \
30
+ --save_total_limit 1 \
31
+ --learning_rate 2e-5 \
32
+ --weight_decay 0. \
33
+ --warmup_ratio 0.03 \
34
+ --lr_scheduler_type "cosine" \
35
+ --logging_steps 1 \
36
+ --tf32 True \
37
+ --model_max_length 2048 \
38
+ --gradient_checkpointing True \
39
+ --dataloader_num_workers 8 \
40
+ --lazy_preprocess True \
41
+ --report_to tensorboard \
42
+ --cache_dir "./cache_dir"
scripts/v1_5/pretrain.sh ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ DATA_ROOT="llava_all_image_video"
3
+ CUDA_VISIBLE_DEVICES=0,1,2,3 deepspeed llava/train/train_mem.py \
4
+ --deepspeed ./scripts/zero2.json \
5
+ --model_name_or_path lmsys/vicuna-7b-v1.5 \
6
+ --version plain \
7
+ --data_path train_json/valley_llavaimage.json \
8
+ --video_folder ${DATA_ROOT} \
9
+ --image_folder ${DATA_ROOT} \
10
+ --X "Video" "Image" \
11
+ --video_tower LanguageBind/LanguageBind_Video \
12
+ --image_tower LanguageBind/LanguageBind_Image \
13
+ --mm_projector_type mlp2x_gelu \
14
+ --tune_mm_mlp_adapter True \
15
+ --mm_vision_select_layer -2 \
16
+ --mm_use_x_start_end False \
17
+ --mm_use_x_patch_token False \
18
+ --bf16 True \
19
+ --output_dir ./checkpoints/Video-LLaVA-Pretrain-7B \
20
+ --num_train_epochs 1 \
21
+ --per_device_train_batch_size 32 \
22
+ --per_device_eval_batch_size 4 \
23
+ --gradient_accumulation_steps 2 \
24
+ --evaluation_strategy "no" \
25
+ --save_strategy "steps" \
26
+ --save_steps 24000 \
27
+ --save_total_limit 1 \
28
+ --learning_rate 1e-3 \
29
+ --weight_decay 0. \
30
+ --warmup_ratio 0.03 \
31
+ --lr_scheduler_type "cosine" \
32
+ --logging_steps 1 \
33
+ --tf32 True \
34
+ --model_max_length 2048 \
35
+ --gradient_checkpointing True \
36
+ --dataloader_num_workers 8 \
37
+ --lazy_preprocess True \
38
+ --report_to tensorboard \
39
+ --cache_dir "./cache_dir"
scripts/zero2.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "train_micro_batch_size_per_gpu": "auto",
14
+ "train_batch_size": "auto",
15
+ "gradient_accumulation_steps": "auto",
16
+ "zero_optimization": {
17
+ "stage": 2,
18
+ "overlap_comm": true,
19
+ "contiguous_gradients": true,
20
+ "sub_group_size": 1e9,
21
+ "reduce_bucket_size": "auto"
22
+ }
23
+ }
scripts/zero3.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "train_micro_batch_size_per_gpu": "auto",
14
+ "train_batch_size": "auto",
15
+ "gradient_accumulation_steps": "auto",
16
+ "zero_optimization": {
17
+ "stage": 3,
18
+ "overlap_comm": true,
19
+ "contiguous_gradients": true,
20
+ "sub_group_size": 1e9,
21
+ "reduce_bucket_size": "auto",
22
+ "stage3_prefetch_bucket_size": "auto",
23
+ "stage3_param_persistence_threshold": "auto",
24
+ "stage3_max_live_parameters": 1e9,
25
+ "stage3_max_reuse_distance": 1e9,
26
+ "stage3_gather_16bit_weights_on_model_save": true
27
+ }
28
+ }
scripts/zero3_offload.json ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "fp16": {
3
+ "enabled": "auto",
4
+ "loss_scale": 0,
5
+ "loss_scale_window": 1000,
6
+ "initial_scale_power": 16,
7
+ "hysteresis": 2,
8
+ "min_loss_scale": 1
9
+ },
10
+ "bf16": {
11
+ "enabled": "auto"
12
+ },
13
+ "optimizer": {
14
+ "type": "AdamW",
15
+ "params": {
16
+ "lr": "auto",
17
+ "betas": "auto",
18
+ "eps": "auto",
19
+ "weight_decay": "auto"
20
+ }
21
+ },
22
+ "scheduler": {
23
+ "type": "WarmupLR",
24
+ "params": {
25
+ "warmup_min_lr": "auto",
26
+ "warmup_max_lr": "auto",
27
+ "warmup_num_steps": "auto"
28
+ }
29
+ },
30
+ "zero_optimization": {
31
+ "stage": 3,
32
+ "offload_optimizer": {
33
+ "device": "cpu",
34
+ "pin_memory": true
35
+ },
36
+ "offload_param": {
37
+ "device": "cpu",
38
+ "pin_memory": true
39
+ },
40
+ "overlap_comm": true,
41
+ "contiguous_gradients": true,
42
+ "sub_group_size": 1e9,
43
+ "reduce_bucket_size": "auto",
44
+ "stage3_prefetch_bucket_size": "auto",
45
+ "stage3_param_persistence_threshold": "auto",
46
+ "stage3_max_live_parameters": 1e9,
47
+ "stage3_max_reuse_distance": 1e9,
48
+ "gather_16bit_weights_on_model_save": true
49
+ },
50
+ "gradient_accumulation_steps": "auto",
51
+ "gradient_clipping": "auto",
52
+ "train_batch_size": "auto",
53
+ "train_micro_batch_size_per_gpu": "auto",
54
+ "steps_per_print": 1e5,
55
+ "wall_clock_breakdown": false
56
+ }