Spaces:
Runtime error
Runtime error
File size: 1,204 Bytes
88768cb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
from abc import ABC, abstractmethod
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS
class TextProcessor(ABC):
@abstractmethod
def split_text(self, text):
pass
@abstractmethod
def create_embeddings(self, chunks):
pass
class DefaultTextProcessor(TextProcessor):
def __init__(self,chunk_size,chunk_overlap):
self.chunk_overlap = chunk_overlap
self.chunk_size = chunk_size
def split_text(self, text):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=self.chunk_size ,
chunk_overlap=self.chunk_overlap,
separators=[" ", ",", "\n"],
length_function=len
)
chunks = text_splitter.split_text(text)
return chunks
def create_embeddings(self, chunks):
if not chunks:
return None
embeddings = OpenAIEmbeddings()
try:
return FAISS.from_texts(chunks, embeddings)
except Exception as e:
print(f"Error creating embeddings: {e}")
return None
|