translate / split.py
Dy3257's picture
Upload 19 files
535a983 verified
raw
history blame
No virus
741 Bytes
def split_string(tokens):
result = []
done = 1
while(len(tokens) > 10 and done):
done = 0
for i in range(10, len(tokens), 1):
if i > 30 :
if tokens[i] in [".", "。", "!", "!", "?", "?", ",", ",", "……"]:
result.append(tokens[:i+1])
tokens = tokens[i+1:]
done = 1
break
else :
if tokens[i] in [".", "。", "!", "!", "?","?"] :
result.append(tokens[:i+1])
tokens = tokens[i+1:]
done = 1
break
if len(tokens) > 0 :
result.append(tokens)
return result