File size: 1,654 Bytes
032e687 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
import json
import os
def load_jsonl(json_file):
with open(json_file) as f:
lines = f.readlines()
data = []
for line in lines:
data.append(json.loads(line))
return data
json_data = "/mnt/bn/xiangtai-training-data/project/VLM/data/SOLO_SFT/all_data.jsonl"
json_data_new = "/mnt/bn/xiangtai-training-data/project/VLM/data/all_data_new.jsonl"
image_data_path = "/mnt/bn/xiangtai-training-data/project/VLM/data/SOLO_SFT/images"
new_json_data = []
a = load_jsonl(json_data)
for index, i in enumerate(a):
conversations = i['conversations']
if 'image' in i.keys() and not os.path.exists(os.path.join(image_data_path, i['image'])):
# print("find", i)
# exit()
print("Missing: ",i)
continue
new_json_data.append(i)
with open(json_data_new, 'w') as f:
json.dump(new_json_data, f)
# print(os.path.join(image_data_path, i['image']))
# if not os.path.exists(os.path.join(image_data_path, i['image'])):
# print(i['images'])
# for msg in conversations:
# if "role" in msg.keys():
# print(i)
# print(index)
# exit()
# elif 'from' in msg.keys():
# continue
# elif 'value' in msg.keys():
# continue
# else:
# print(msg.keys)
# if msg['from'] == 'human' or msg['from'] == 'user' or msg['role'] == 'user':
# continue
# elif msg['from'] == 'gpt' or msg['from'] == 'model' or msg['role'] == 'assistant':
# continue
# for item in conversations:
# if type(item) is str:
# print(conversations)
|