no message

main.py

@@ -112,33 +112,59 @@ def segment_text(text: str, max_tokens=500):  # Slightly less than 512 for safet
     # Use spaCy to divide the document into sentences
     doc = nlp(text)
     sentences = [sent.text.strip() for sent in doc.sents]
-    
+
     segments = []
     current_segment = []
     current_length = 0
     
     for sentence in sentences:
-        sentence_length = len(sentence.split())  # Simple word count
+        sentence_words = sentence.split()
+        sentence_length = len(sentence_words)
+        
+        # If sentence exceeds max_tokens, split it further
+        if sentence_length > max_tokens:
+            parts = split_into_parts(sentence, max_tokens)
+            segments.extend(parts)  # Add split parts directly to segments
+            continue
+        
         if current_length + sentence_length > max_tokens:
-            if current_segment:  # Make sure there's something to add
-                segments.append(' '.join(current_segment))
+            segments.append(' '.join(current_segment))
             current_segment = [sentence]
             current_length = sentence_length
         else:
             current_segment.append(sentence)
             current_length += sentence_length
     
-    # Add the last segment if any
-    if current_segment:
+    if current_segment:  # Add the last segment if any
         segments.append(' '.join(current_segment))
     
     return segments
 
+def split_into_parts(text, max_tokens):
+    words = text.split()
+    parts = []
+    for i in range(0, len(words), max_tokens):
+        part = " ".join(words[i:i + max_tokens])
+        parts.append(part)
+    return parts
+
+
 
 classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
 
 def classify_segments(segments):
-    return [classifier(segment) for segment in segments]
+    results = []
+    for segment in segments:
+        try:
+            if len(segment.split()) <= 512:  # Ensure segment is within the limit
+                result = classifier(segment)
+                results.append(result)
+            else:
+                results.append({"error": f"Segment too long: {len(segment.split())} tokens"})
+        except Exception as e:
+            results.append({"error": str(e)})
+    return results
+
 
 
 @app.post("/summarize")