File size: 2,131 Bytes
b1e25b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
import json
import argparse
def main():
parser = argparse.ArgumentParser(description='Convert and merge JSONL files with question-answer mappings')
parser.add_argument('orig_path', help='Path to the original JSONL file')
parser.add_argument('out_path', help='Path to the output JSONL file')
parser.add_argument('mapping_paths', nargs='+', help='Path(s) to mapping JSONL file(s)')
args = parser.parse_args()
# Original data file paths from command line arguments
orig_path = args.orig_path
out_path = args.out_path
mapping_paths = args.mapping_paths
# Step 1: Build question -> {context, answers} mapping
mapping = {}
for mp in mapping_paths:
with open(mp, 'r', encoding='utf-8') as f_map:
for idx, line in enumerate(f_map):
obj = json.loads(line)
q = obj.get("question")
if q is None:
continue
# Ensure we get the context
ctx = obj.get("context", "")
# Some files have "answer" field, some have "answers"
raw_ans = obj.get("answers", obj.get("answer", []))
# Normalize answer(s) to list format
if isinstance(raw_ans, list):
ans = raw_ans
else:
ans = [raw_ans]
# If the same question appears in multiple mapping files, later ones will overwrite earlier ones
mapping[q] = {"context": ctx, "answers": ans}
# Step 2: Read original file, perform replacement and write output
with open(orig_path, 'r', encoding='utf-8') as f_in, \
open(out_path, 'w', encoding='utf-8') as f_out:
for line in f_in:
item = json.loads(line)
inp = item.get("input")
if inp in mapping:
item["context"] = mapping[inp]["context"]
item["answers"] = mapping[inp]["answers"]
f_out.write(json.dumps(item, ensure_ascii=False) + "\n")
print(f"Merge completed, output file: {out_path}")
if __name__ == "__main__":
main() |