mehran commited on
Commit
576f1fa
1 Parent(s): be62ffe

Update model.py

Browse files
Files changed (1) hide show
  1. model.py +1 -1
model.py CHANGED
@@ -451,7 +451,7 @@ class KenlmModel:
451
  if normalize_numbers:
452
  norm_list += [normalizers.Replace(Regex("[۱۲۳۴۵۶۷۸۹]"), "۰")]
453
  if remove_puctuation:
454
- norm_list += [normalizers.Replace(Regex("[.!؛،؟]"), "")]
455
  if remove_non_farsi:
456
  norm_list += [normalizers.Replace(Regex("[^\u060c\u061b\u061f\u0622\u0623\u0624\u0626\u0627"
457
  "\u0628\u062a\u062b\u062c\u062d\u062e\u062f\u0630\u0631"
 
451
  if normalize_numbers:
452
  norm_list += [normalizers.Replace(Regex("[۱۲۳۴۵۶۷۸۹]"), "۰")]
453
  if remove_puctuation:
454
+ norm_list += [normalizers.Replace(Regex("[\\.!؛،؟]"), "")]
455
  if remove_non_farsi:
456
  norm_list += [normalizers.Replace(Regex("[^\u060c\u061b\u061f\u0622\u0623\u0624\u0626\u0627"
457
  "\u0628\u062a\u062b\u062c\u062d\u062e\u062f\u0630\u0631"