todset.py

import json
from tqdm import tqdm

lang = ""

with open(f"dset{lang}.txt", "r") as f:
    lines = [x.rstrip("\n").lower().split("→") for x in tqdm(f.readlines())]
    lines = [(x[0].replace("\\n", "\n"), x[1].replace("\\n", "\n")) for x in lines]

responses = []
for i in tqdm(lines):
    if i[1] not in responses:
        responses.append(i[1])

dset = {}
for sample in tqdm(lines):
    dset[sample[0]] = responses.index(sample[1])

with open(f"dataset{lang}.json", "w") as f:
    json.dump(dset, f, ensure_ascii=False)

with open(f"responses{lang}.txt", "w") as f:
    for i in tqdm(responses):
        f.write(i+"\n")