Commit
•
43e4326
1
Parent(s):
1a1a985
bug fix
Browse files
ud.py
CHANGED
|
@@ -78,7 +78,7 @@ class MecabPreTokenizer(MecabTokenizer):
|
|
| 78 |
else:
|
| 79 |
e=s+len(c)
|
| 80 |
z.append((s,e))
|
| 81 |
-
return [normalized_string[s:e] for s,e in z]
|
| 82 |
def pre_tokenize(self,pretok):
|
| 83 |
pretok.split(self.mecab_split)
|
| 84 |
|
| 78 |
else:
|
| 79 |
e=s+len(c)
|
| 80 |
z.append((s,e))
|
| 81 |
+
return [normalized_string[s:e] for s,e in z if e>0]
|
| 82 |
def pre_tokenize(self,pretok):
|
| 83 |
pretok.split(self.mecab_split)
|
| 84 |
|