nickmuchi commited on
Commit
e85d7fc
1 Parent(s): 45412ff

Update functions.py

Browse files
Files changed (1) hide show
  1. functions.py +2 -2
functions.py CHANGED
@@ -337,8 +337,8 @@ def chunk_and_preprocess_text(text, model_name= 'philschmid/flan-t5-base-samsum'
337
  count = -1
338
 
339
  for sentence in sentences:
340
- count += 1
341
- combined_length = len(tokenizer.tokenize(sentence)) + length # add the no. of sentence tokens to the length counter
342
 
343
  if combined_length <= tokenizer.max_len_single_sentence: # if it doesn't exceed
344
  chunk += sentence + " " # add the sentence to the chunk
 
337
  count = -1
338
 
339
  for sentence in sentences:
340
+ count += 1
341
+ combined_length = len(tokenizer.tokenize(sentence)) + length # add the no. of sentence tokens to the length counter
342
 
343
  if combined_length <= tokenizer.max_len_single_sentence: # if it doesn't exceed
344
  chunk += sentence + " " # add the sentence to the chunk