pleonova commited on
Commit
d9b3880
1 Parent(s): 4d40c32

Remove nlp() from nested_sentences

Browse files
Files changed (1) hide show
  1. models.py +0 -1
models.py CHANGED
@@ -10,7 +10,6 @@ def create_nest_sentences(document:str, token_max_length = 1024):
10
  sent = []
11
  length = 0
12
  tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-mnli')
13
- tokens = nlp(document)
14
 
15
  for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", ' ')):
16
  tokens_in_sentence = tokenizer(str(sentence), truncation=False, padding=False)[0] # hugging face transformer tokenizer
10
  sent = []
11
  length = 0
12
  tokenizer = AutoTokenizer.from_pretrained('facebook/bart-large-mnli')
 
13
 
14
  for sentence in re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', document.replace("\n", ' ')):
15
  tokens_in_sentence = tokenizer(str(sentence), truncation=False, padding=False)[0] # hugging face transformer tokenizer