| #!/usr/bin/env python3 | |
| import sys | |
| from typing import Dict | |
| def load_tokens(): | |
| ans = dict() | |
| i = 0 | |
| with open("tokens.txt", encoding="utf-8") as f: | |
| for line in f: | |
| if len(line.strip().split()) == 2: | |
| sys.exit("Already converted!\nExiting") | |
| ans[i] = line.strip() | |
| i += 1 | |
| return ans | |
| def write_tokens(tokens: Dict[int, str]): | |
| with open("new_tokens.txt", "w", encoding="utf-8") as f: | |
| for idx, s in tokens.items(): | |
| f.write(f"{s} {idx}\n") | |
| def main(): | |
| tokens = load_tokens() | |
| write_tokens(tokens) | |
| if __name__ == "__main__": | |
| main() | |