txya900619 commited on
Commit
576392b
1 Parent(s): 5ee043f

fix: jieba dict update encoding ascii bug

Browse files
Files changed (1) hide show
  1. ipa/proc_text.py +1 -1
ipa/proc_text.py CHANGED
@@ -25,7 +25,7 @@ def update_jieba_dict(
25
  jieba_dict_path.unlink(missing_ok=True)
26
  Path("/tmp/jieba.cache").unlink(missing_ok=True)
27
 
28
- with jieba_dict_path.open("w") as file:
29
  for word in lexicon:
30
  if word in high_freq_words:
31
  file.write(f"{word} {len(word) * high_freq_words_weight}\n")
 
25
  jieba_dict_path.unlink(missing_ok=True)
26
  Path("/tmp/jieba.cache").unlink(missing_ok=True)
27
 
28
+ with jieba_dict_path.open("w", encoding="utf-8") as file:
29
  for word in lexicon:
30
  if word in high_freq_words:
31
  file.write(f"{word} {len(word) * high_freq_words_weight}\n")