Spaces:
Runtime error
Runtime error
| from abc import ABC, abstractmethod | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.embeddings.openai import OpenAIEmbeddings | |
| from langchain.vectorstores import FAISS | |
| class TextProcessor(ABC): | |
| def split_text(self, text): | |
| pass | |
| def create_embeddings(self, chunks): | |
| pass | |
| class DefaultTextProcessor(TextProcessor): | |
| def __init__(self,chunk_size,chunk_overlap): | |
| self.chunk_overlap = chunk_overlap | |
| self.chunk_size = chunk_size | |
| def split_text(self, text): | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=self.chunk_size , | |
| chunk_overlap=self.chunk_overlap, | |
| separators=[" ", ",", "\n"], | |
| length_function=len | |
| ) | |
| chunks = text_splitter.split_text(text) | |
| return chunks | |
| def create_embeddings(self, chunks): | |
| if not chunks: | |
| return None | |
| embeddings = OpenAIEmbeddings() | |
| try: | |
| return FAISS.from_texts(chunks, embeddings) | |
| except Exception as e: | |
| print(f"Error creating embeddings: {e}") | |
| return None | |