File size: 1,204 Bytes
88768cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
from abc import ABC, abstractmethod
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS

class TextProcessor(ABC):

    @abstractmethod
    def split_text(self, text):
        pass

    @abstractmethod
    def create_embeddings(self, chunks):
        pass


class DefaultTextProcessor(TextProcessor):
    

    def __init__(self,chunk_size,chunk_overlap):
        self.chunk_overlap = chunk_overlap
        self.chunk_size = chunk_size

    def split_text(self, text):
       
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=self.chunk_size , 
            chunk_overlap=self.chunk_overlap, 
            separators=[" ", ",", "\n"],
            length_function=len
        )

        chunks = text_splitter.split_text(text)
        return chunks

    def create_embeddings(self, chunks):
        
        if not chunks:
            return None
        embeddings = OpenAIEmbeddings()
        try:
            return FAISS.from_texts(chunks, embeddings)
        except Exception as e:
            print(f"Error creating embeddings: {e}")
            return None