amank commited on
Commit
7007f93
1 Parent(s): 7839b8e

Minor Code cleanup in utils file

Browse files
Files changed (1) hide show
  1. utils.py +2 -9
utils.py CHANGED
@@ -42,15 +42,8 @@ def keep_devnagri_hf_doc(document):
42
  return cleaned_text
43
 
44
  if batched:
45
- text_ls = document['text']
46
- cleaned_text_ls = []
47
- for text in text_ls:
48
- cleaned_text = get_clean_text(text)
49
- cleaned_text_ls.append(cleaned_text)
50
- document['text'] = cleaned_text_ls
51
  else:
52
- text = document['text']
53
- cleaned_text = get_clean_text(text)
54
- document['text'] = cleaned_text
55
 
56
  return document
 
42
  return cleaned_text
43
 
44
  if batched:
45
+ document['text'] = [get_clean_text(text) for text in document['text']]
 
 
 
 
 
46
  else:
47
+ document['text'] = get_clean_text(document['text'])
 
 
48
 
49
  return document