no message

main.py

@@ -81,45 +81,29 @@ async def generate_text(item: Item):
     # Stream response back to the client
     return StreamingResponse(generate_stream(item), media_type="application/x-ndjson")
 
-
-
 def split_text_by_tokens(text, max_tokens=1024):
-    # Tokenize the text
     print("Tokenizing text...")
-
     tokens = tokenizer.tokenize(text)
-    # Split into chunks of max_tokens
+    
+    chunks = []
+    token_counts = []
+    
     for i in range(0, len(tokens), max_tokens):
-        # Ensure not to exceed the token limit
-        yield tokenizer.convert_tokens_to_string(tokens[i:i+max_tokens])
-
-def summarize_large_text(text):
-    # Use the updated split_text_by_tokens function
-    chunks = list(split_text_by_tokens(text, max_tokens=1024 - 10))  # Slight buffer to avoid edge cases
-    summaries = []
-    print("Tokenization complete, summarizing chunks...")
-
-    for chunk in chunks:
-        print("loop chunks...")
-
-        # Check if chunk is within the token limit just to be sure
-        chunk_tokens = tokenizer.encode(chunk)
-        if len(chunk_tokens) > 1024:
-            continue  # Skip chunks that are still too large
-        # Perform summarization on the chunk
-        summary = summarizer(chunk, max_length=500, min_length=100, do_sample=False)
-        if summary:
-            summaries.append(summary[0]['summary_text'])
-    combined_summary = ' '.join(summaries)
-    return combined_summary
+        chunk = tokenizer.convert_tokens_to_string(tokens[i:i+max_tokens])
+        chunks.append(chunk)
+        token_counts.append(len(tokenizer.encode(chunk)))  # Count tokens of the current chunk
+
+    print("Tokenization complete.")
+    return chunks, token_counts
 
 @app.post("/summarize")
 async def summarize_text(request: SummarizeRequest):
     try:
-        summarized_text = summarize_large_text(request.text)
-        return JSONResponse(content={"summary": summarized_text})
+        chunks, token_counts = split_text_by_tokens(request.text, max_tokens=1024 - 10)  # Slight buffer to avoid edge cases
+        chunk_data = [{'chunk': chunk, 'tokens': count} for chunk, count in zip(chunks, token_counts)]
+        return JSONResponse(content={"chunks": chunk_data})
     except Exception as e:
-        print(f"Error during summarization: {e}")
+        print(f"Error during tokenization: {e}")
         raise HTTPException(status_code=500, detail=str(e))