Marroco93 commited on
Commit
131731f
·
1 Parent(s): 266a737

no message

Browse files
Files changed (1) hide show
  1. main.py +1 -0
main.py CHANGED
@@ -140,6 +140,7 @@ def segment_text(text: str, max_tokens=500): # Setting a conservative limit bel
140
 
141
 
142
  tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
 
143
 
144
  def robust_segment_text(text: str, max_tokens=510): # Slightly less to ensure a buffer
145
  doc = nlp(text)
 
140
 
141
 
142
  tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
143
+ classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
144
 
145
  def robust_segment_text(text: str, max_tokens=510): # Slightly less to ensure a buffer
146
  doc = nlp(text)