There seems to be something from about how we handle loading the cached embedding model and how the information is reported in the terminal, check the code: import os
import nltk
import torch
import pickle
import chardet
import numpy as np
import threading
from transformers import AutoTokenizer, AutoModel, logging
from sklearn.metrics.pairwise import cosine_similarity

# Download Punkt sentence tokenizer if not already downloaded
nltk.download('punkt', quiet=True)

# Suppress warnings from transformers
logging.set_verbosity_error()

# Set up cache directory for models and embeddings
cache_dir = os.path.join(os.path.dirname(__file__), "model_cache")
print(f"cache directory: {cache_dir}")
os.makedirs(cache_dir, exist_ok=True)

# ---- Embedding Model ----
def meanpooling(output, mask):
    embeddings = output[0]
    mask = mask.unsqueeze(-1).expand(embeddings.size()).float()
    return torch.sum(embeddings * mask, 1) / torch.clamp(mask.sum(1), min=1e-9)

def load_embedding_model(model_name="sentence-transformers/all-MiniLM-L6-v2"):
    """
    Loads the embedding model and tokenizer from cache or downloads if not present.
    """
    model_path = os.path.join(cache_dir, model_name.replace("/", "_"))
    try:
        if os.path.exists(os.path.join(model_path, "pytorch_model.bin")):
            print("Loading embedding model from cache...")
            tokenizer = AutoTokenizer.from_pretrained(model_path)
            model = AutoModel.from_pretrained(model_path)
        else:
            print("Downloading embedding model...")
            tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir=model_path)
            model = AutoModel.from_pretrained(model_name, cache_dir=model_path)
            tokenizer.save_pretrained(model_path)
            model.save_pretrained(model_path)
    except Exception as e:
        print(f"Error loading embedding model: {str(e)}")
        raise
    return tokenizer, model

# Load the embedding model and tokenizer
tokenizer, model = load_embedding_model()
device = torch.device("cpu")
model.to(device)

def get_embedding(texts, batch_size=32):
    """
    Calculates embeddings for a list of texts in batches.
    """
    all_embeddings = []
    try:
        for i in range(0, len(texts), batch_size):
            batch = texts[i : i + batch_size]
            inputs = tokenizer(batch, padding=True, truncation=True, return_tensors='pt').to(device)
            with torch.no_grad():
                output = model(**inputs)
            batch_embeddings = meanpooling(output, inputs['attention_mask']).cpu().numpy()
            all_embeddings.append(batch_embeddings)
    except Exception as e:
        print(f"Error generating embeddings: {str(e)}")
        raise
    return np.concatenate(all_embeddings, axis=0)

# ---- Search Engine ----
class LocalSemanticSearchEngine:
    def __init__(self, embeddings_file="embeddings.pkl"):
        self.corpus = {}
        self.embeddings = {}
        self.embeddings_file = embeddings_file
        self.lock = threading.Lock()  # For thread safety

    def load_documents(self, path):
        try:
            if os.path.isdir(path):
                self._load_embeddings_from_disk(path)
                files = [f for f in os.listdir(path) if f.lower().endswith((".txt", ".pdf", ".docx", ".pptx", ".rtf"))]
                total_files = len(files)
                for i, filename in enumerate(files):
                    file_path = os.path.join(path, filename)
                    if file_path not in self.embeddings:
                        try:
                            self.load_document(file_path)
                        except Exception as e:
                            print(f"Error loading document {filename}: {str(e)}")
                self._save_embeddings_to_disk(path)
            elif os.path.isfile(path) and path.lower().endswith((".txt", ".pdf", ".docx", ".pptx", ".rtf")):
                self.load_document(path)
                self._save_embeddings_to_disk(os.path.dirname(path))
            else:
                raise ValueError("Invalid path provided. Please provide a supported file or a directory.")
        except Exception as e:
            print(f"Error loading documents: {str(e)}")
            raise

    def load_document(self, file_path):
        try:
            _, ext = os.path.splitext(file_path)
            ext = ext.lower()

            if ext == ".txt":
                text = self.read_text_file(file_path)
            elif ext == ".pdf":
                text = self.extract_text_from_pdf(file_path)
            elif ext == ".docx":
                text = self.extract_text_from_docx(file_path)
            elif ext == ".pptx":
                text = self.extract_text_from_pptx(file_path)
            elif ext == ".rtf":
                text = self.read_text_file(file_path)  # RTF can be treated as text
            else:
                raise ValueError(f"Unsupported file type: {ext}")

            if not text.strip():
                print(f"Warning: No text content found in {file_path}")
                return

            sentences = nltk.sent_tokenize(text)
            if not sentences:
                print(f"Warning: No sentences extracted from {file_path}")
                return

            embeddings = get_embedding(sentences)
            if embeddings.size == 0:
                print(f"Warning: No embeddings generated for {file_path}")
                return

            self.corpus[file_path] = sentences
            self.embeddings[file_path] = embeddings
        except Exception as e:
            print(f"Error loading document '{file_path}': {str(e)}")

    def read_text_file(self, file_path):
        try:
            with open(file_path, 'rb') as file:
                raw_data = file.read()

            result = chardet.detect(raw_data)
            encoding = result['encoding'] or 'utf-8'

            with open(file_path, 'r', encoding=encoding, errors='ignore') as file:
                content = file.read()
            return content
        except Exception as e:
            print(f"Error reading file '{file_path}': {e}")
            return ""

    def extract_text_from_pdf(self, file_path):
        try:
            from PyPDF2 import PdfReader
            reader = PdfReader(file_path)
            text = ''
            for page in reader.pages:
                page_text = page.extract_text()
                if page_text:
                    text += page_text
            return text
        except Exception as e:
            print(f"Error extracting text from pdf '{file_path}': {str(e)}")
            raise

    def extract_text_from_docx(self, file_path):
        try:
            from docx import Document
            doc = Document(file_path)
            return "\n".join([paragraph.text for paragraph in doc.paragraphs])
        except Exception as e:
            print(f"Error extracting text from docx '{file_path}': {str(e)}")
            raise

    def extract_text_from_pptx(self, file_path):
        try:
            from pptx import Presentation
            prs = Presentation(file_path)
            text = []
            for slide in prs.slides:
                for shape in slide.shapes:
                    if hasattr(shape, "text"):
                        text.append(shape.text)
            return "\n".join(text)
        except Exception as e:
            print(f"Error extracting text from pptx '{file_path}': {str(e)}")
            raise

    def _load_embeddings_from_disk(self, path):
        try:
            embeddings_path = os.path.join(path, self.embeddings_file)
            if os.path.exists(embeddings_path):
                with open(embeddings_path, "rb") as f:
                    self.embeddings = pickle.load(f)
                    self.corpus = {}
                    for file_path in self.embeddings.keys():
                        text = self.read_text_file(file_path)
                        if text.strip():
                            sentences = nltk.sent_tokenize(text)
                            self.corpus[file_path] = sentences
        except Exception as e:
            print(f"Error loading embeddings from disk: {str(e)}")
            # Continue without raising an exception to allow new embeddings to be created

    def _save_embeddings_to_disk(self, path):
        try:
            embeddings_path = os.path.join(path, self.embeddings_file)
            with self.lock:
                with open(embeddings_path, "wb") as f:
                    pickle.dump(self.embeddings, f)
        except Exception as e:
            print(f"Error saving embeddings to disk: {str(e)}")
            # Continue without raising an exception

    def search(self, query, similarity_threshold=0.4):
        try:
            if not self.embeddings:
                print("Embeddings are empty. Please load documents first.")
                return None, 0.0

            query_embedding = get_embedding([query])[0]
            max_similarity = -1
            most_similar_file = None

            for file_path, file_embeddings in self.embeddings.items():
                similarities = cosine_similarity(query_embedding.reshape(1, -1), file_embeddings)
                max_sim = np.max(similarities)
                if max_sim > max_similarity:
                    max_similarity = max_sim
                    most_similar_file = file_path

            if max_similarity < similarity_threshold:
                return None, max_similarity

            return most_similar_file, max_similarity
        except Exception as e:
            print(f"Error searching for similar documents: {str(e)}")
            raise

# ---- RAG Enhanced LLM Inference ----
class RAGEnabledLLM:
    def __init__(self, llm, search_engine, max_token_length):
        self.llm = llm
        self.search_engine = search_engine
        self.max_token_length = int(max_token_length)
        self.response = ""
        self.max_context_length = 1024
        self.last_source_file = None  # To store the source file used

    def generate_response(self, query, context_length, temperature=0.7, top_p=0.9, top_k=40, similarity_threshold=0.4):
        try:
            most_similar_file, similarity_score = self.search_engine.search(query, similarity_threshold=similarity_threshold)

            if most_similar_file:
                with open(most_similar_file, 'r', encoding='utf-8', errors='ignore') as file:
                    context = file.read()
                context = context[:min(
                    int(context_length), self.max_context_length - len(query) - 100)]
                self.last_source_file = most_similar_file
            else:
                context = ""
                self.last_source_file = None

            prompt = f"""
            You are a helpful and informative AI assistant.
            Context: '{context}'

            Question: {query}

            Answer the question based on the context and your prior knowledge.
            If you don't have enough information to answer the question, say that.
            After your answer, provide 3 follow-up questions that are relevant to the context and the user's original question.

            You MUST format your response as a valid JSON object EXACTLY like this:

            {{
                "answer": "Your answer here.",
                "follow_up_questions": [
                    "Question 1",
                    "Question 2",
                    "Question 3"
                ]
            }}

            Do NOT include anything else in your response.
            """

            # Ensure context_length is an integer:
            if not isinstance(context_length, int):
                raise TypeError(f"context_length must be an integer, got {type(context_length)}")

            # Modify this part to yield tokens one by one
            params = {
                'max_tokens': self.max_token_length,
                'stop': ["
"],
                'echo': False,
                'stream': True,
                'temperature': temperature,
                'top_p': top_p,
                'top_k': top_k
            }
            for token in self.llm(prompt, **params):
                yield token['choices'][0]['text']
        except Exception as e:
            print(f"Error generating response with RAG: {str(e)}")
            raise

    def generate_response_without_rag(self, query, context_length, temperature=0.7, top_p=0.9, top_k=40):
        try:
            prompt = f"""
            You are a helpful and informative AI assistant.

            Question: {query}

            Answer the question based on your own knowledge.
            If you don't have enough information to answer the question, say that.
            After your answer, provide 3 follow-up questions that are relevant to the user's original question.

            You MUST format your response as a valid JSON object EXACTLY like this:

            {{
                "answer": "Your answer here.",
                "follow_up_questions": [
                    "Question 1",
                    "Question 2",
                    "Question 3"
                ]
            }}

            Do NOT include anything else in your response.
            """

            # Ensure context_length is an integer:
            if not isinstance(context_length, int):
                raise TypeError(f"context_length must be an integer, got {type(context_length)}")

            # Yield tokens one by one
            params = {
                'max_tokens': self.max_token_length,
                'stop': ["
"],
                'echo': False,
                'stream': True,
                'temperature': temperature,
                'top_p': top_p,
                'top_k': top_k
            }
            for token in self.llm(prompt, **params):
                yield token['choices'][0]['text']
        except Exception as e:
            print(f"Error generating response without RAG: {str(e)}")
            raise