amielle commited on
Commit
6bc4ea4
1 Parent(s): 7d0a303

fix: Update handler for text length

Browse files
Files changed (1) hide show
  1. util/textproc.py +3 -1
util/textproc.py CHANGED
@@ -52,7 +52,9 @@ def get_word_index(s, limit):
52
  words = re.findall(r'\s*\S+\s*', s)
53
  return sum(map(len, words[:limit])) + len(words[limit]) - len(words[limit].lstrip())
54
  except:
55
- return len(s)
 
 
56
 
57
 
58
  def post_process(s):
 
52
  words = re.findall(r'\s*\S+\s*', s)
53
  return sum(map(len, words[:limit])) + len(words[limit]) - len(words[limit].lstrip())
54
  except:
55
+ l = len(s)
56
+ chr_limit = 3500
57
+ return l if l < chr_limit else chr_limit
58
 
59
 
60
  def post_process(s):