nickmuchi commited on
Commit
42a3a84
1 Parent(s): 1c1d7c0

Update functions.py

Browse files
Files changed (1) hide show
  1. functions.py +11 -11
functions.py CHANGED
@@ -346,19 +346,19 @@ def chunk_and_preprocess_text(text, model_name= 'philschmid/flan-t5-base-samsum'
346
  chunk += sentence + " " # add the sentence to the chunk
347
  length = combined_length # update the length counter
348
 
349
- # if it is the last sentence
350
- if count == len(sentences) - 1:
351
- chunks.append(chunk) # save the chunk
352
 
353
- else:
354
- chunks.append(chunk) # save the chunk
355
- # reset
356
- length = 0
357
- chunk = ""
358
 
359
- # take care of the overflow sentence
360
- chunk += sentence + " "
361
- length = len(tokenizer.tokenize(sentence))
362
 
363
  return chunks
364
 
 
346
  chunk += sentence + " " # add the sentence to the chunk
347
  length = combined_length # update the length counter
348
 
349
+ # if it is the last sentence
350
+ if count == len(sentences) - 1:
351
+ chunks.append(chunk) # save the chunk
352
 
353
+ else:
354
+ chunks.append(chunk) # save the chunk
355
+ # reset
356
+ length = 0
357
+ chunk = ""
358
 
359
+ # take care of the overflow sentence
360
+ chunk += sentence + " "
361
+ length = len(tokenizer.tokenize(sentence))
362
 
363
  return chunks
364