File size: 1,489 Bytes
16b5280 7fa3a69 16b5280 7fa3a69 16b5280 7fa3a69 16b5280 7fa3a69 16b5280 1d413b9 16b5280 1d413b9 16b5280 7fa3a69 16b5280 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
import os
import json
import collections
def read_qs():
qs, qas = [], []
directory = "./questions"
filenames = [
'math_question.txt',
'qa_question.txt',
'summarization_question.txt',
]
for filename in filenames:
with open(f"{directory}/{filename}", "r", encoding="utf-8") as f:
for idx,line in enumerate(f):
qs.append(line.replace("ηζζθ¦","ηζδΈζζθ¦"))
print(f"read {len(qs)} queries from files")
return qs
def read_qas():
qas = []
directory = "./questions"
for filename in os.listdir(directory):
if filename.endswith(".json") and "qas" in filename:
with open(f"{directory}/{filename}", "r", encoding="utf-8") as f:
for qa in json.loads(f.read()):
qas.append(qa)
print(f"read {len(qas)} query-answers from files")
return qas
def merge(qs, qas):
q_to_as = collections.defaultdict(lambda:[])
for qa in qas:
q_to_as[qa["q"]].append(qa["a"])
qas = []
for q in qs:
if len(q_to_as[q])==0:
continue
a = q_to_as[q].pop()
qas.append({"q":q, "a":a})
print(f"merge {len(qas)} query-answers from files")
return qas
if __name__ == "__main__":
qs = read_qs()
qas = read_qas()
qas = merge(qs, qas)
with open("./questions/qas.json", "w", encoding="utf-8") as f:
f.write(json.dumps(qas, ensure_ascii=False, indent=2)) |