Update main.py

main.py

@@ -6,24 +6,25 @@ from fastapi.middleware.cors import CORSMiddleware
 from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String
 from databases import Database
 from textblob import TextBlob
-import os
 
 import whisperx
 import gc
-import re
-import openai
-import time
 
-import spacy
-
-# Load the English tokenizer, tagger, parser, NER, and word vectors
-nlp = spacy.load("en_core_web_sm")
 
+import nltk
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+import openai
+import time
+nltk.download('punkt')
+nltk.download('stopwords')
 
 openai.api_key = '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'
 
 app = FastAPI()
 
+import os
+
 import requests
 import json
 
@@ -90,12 +91,9 @@ def parse_conversation(content):
 
 
 def extract_active_words(text):
-    # Process the text with spaCy
-    doc = nlp(text)
-    
-    # Extract tokens that are not stopwords
-    active_words = [re.sub(r'[^\w\s]', '', token.text) for token in doc if not token.is_stop and not token.is_digit and not token.is_punct and token.text != "Speaker"]
-    
+    tokens = word_tokenize(text)
+    stop_words = set(stopwords.words('english'))
+    active_words = [word for word in tokens if word.isalnum() and word.lower() not in stop_words]
     return active_words