File size: 741 Bytes
50d6551 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
#!/usr/bin/env python3
import sys
from typing import Dict
def load_tokens():
ans = dict()
i = 0
with open("tokens.json", encoding="utf-8") as f:
for line in f:
line = line.strip()
if '[' in line: continue
if ']' in line: continue
if '"' in line and ',' in line:
line = line[1:-2]
ans[i] = line.strip()
i += 1
print('num tokens', i)
return ans
def write_tokens(tokens: Dict[int, str]):
with open("tokens.txt", "w", encoding="utf-8") as f:
for idx, s in tokens.items():
f.write(f"{s} {idx}\n")
def main():
tokens = load_tokens()
write_tokens(tokens)
if __name__ == "__main__":
main()
|