UNIST-Eunchan commited on
Commit
b48fa68
1 Parent(s): d872090

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -1
app.py CHANGED
@@ -1,6 +1,6 @@
1
  import transformers
2
  import streamlit as st
3
-
4
  from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
5
  import json
6
 
@@ -33,6 +33,7 @@ def infer(input_ids, max_length, temperature, top_k, top_p):
33
 
34
 
35
  def chunking(book_text):
 
36
  segments = []
37
  #sentences, token_lens
38
  current_segment = ""
 
1
  import transformers
2
  import streamlit as st
3
+ from nltk import sent_tokenize
4
  from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
5
  import json
6
 
 
33
 
34
 
35
  def chunking(book_text):
36
+ sentences = sent_tokenize(book_text)
37
  segments = []
38
  #sentences, token_lens
39
  current_segment = ""