File size: 552 Bytes
577164e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 |
from kiwipiepy import Kiwi
kiwi = Kiwi()
def text_tokenize(src):
token_list = kiwi.tokenize(src, normalize_coda=True)
results=''
for token in token_list:
results += f" {token.form}"
return results
def tokenize(src):
token_list = kiwi.tokenize(src, normalize_coda=True)
results=[]
for token in token_list:
results.append(token.form)
return results
def main():
while True:
text = input('>')
result = text_tokenize(text)
print(result)
if __name__ == "__main__":
main()
|