File size: 1,100 Bytes
032e687 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import json
def load_jsonl(json_file):
with open(json_file) as f:
lines = f.readlines()
data = []
for line in lines:
data.append(json.loads(line))
return data
json_data = "/mnt/bn/xiangtai-training-data/project/VLM/data/SOLO_SFT/all_data.jsonl"
image_data = "/mnt/bn/xiangtai-training-data/project/VLM/data/SOLO_SFT/images"
a = load_jsonl(json_data)
for index, i in enumerate(a):
conversations = i['conversations']
image_name = i['image']
for msg in conversations:
if "role" in msg.keys():
print(i)
print(index)
exit()
elif 'from' in msg.keys():
continue
elif 'value' in msg.keys():
continue
else:
print(msg.keys)
# if msg['from'] == 'human' or msg['from'] == 'user' or msg['role'] == 'user':
# continue
# elif msg['from'] == 'gpt' or msg['from'] == 'model' or msg['role'] == 'assistant':
# continue
# for item in conversations:
# if type(item) is str:
# print(conversations)
|