Spaces:
Running
on
Zero
Running
on
Zero
txya900619
commited on
Commit
•
576392b
1
Parent(s):
5ee043f
fix: jieba dict update encoding ascii bug
Browse files- ipa/proc_text.py +1 -1
ipa/proc_text.py
CHANGED
@@ -25,7 +25,7 @@ def update_jieba_dict(
|
|
25 |
jieba_dict_path.unlink(missing_ok=True)
|
26 |
Path("/tmp/jieba.cache").unlink(missing_ok=True)
|
27 |
|
28 |
-
with jieba_dict_path.open("w") as file:
|
29 |
for word in lexicon:
|
30 |
if word in high_freq_words:
|
31 |
file.write(f"{word} {len(word) * high_freq_words_weight}\n")
|
|
|
25 |
jieba_dict_path.unlink(missing_ok=True)
|
26 |
Path("/tmp/jieba.cache").unlink(missing_ok=True)
|
27 |
|
28 |
+
with jieba_dict_path.open("w", encoding="utf-8") as file:
|
29 |
for word in lexicon:
|
30 |
if word in high_freq_words:
|
31 |
file.write(f"{word} {len(word) * high_freq_words_weight}\n")
|