there is a other way of using retrival for the knowlage base and the arxiv thing. use this method instead. update my code with this new retrival thing implemented:
# -*- coding: utf-8 -*-
"""Another copy of Contextual-Retrieval-updated-prompt.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1hUi7ECRU5fXUZ_9IHN5vMOZz7rVO0Qn-
"""

pip install langchain langchain-openai openai faiss-cpu python-dotenv rank_bm25

pip install -U langchain-community

import hashlib
import os
import getpass
from typing import List, Tuple
from dotenv import load_dotenv
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.schema import Document
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain_openai import ChatOpenAI
from langchain.prompts import ChatPromptTemplate
from rank_bm25 import BM25Okapi

from google.colab import userdata
os.environ["OPENAI_API_KEY"] =userdata.get('openai')

class ContextualRetrieval:
    """
    A class that implements the Contextual Retrieval system.
    """

    def __init__(self):
        """
        Initialize the ContextualRetrieval system.
        """
        self.text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=800,
            chunk_overlap=100,
        )
        self.embeddings = OpenAIEmbeddings()
        self.llm = ChatOpenAI(
            model="gpt-4o",
            temperature=0,
            max_tokens=None,
            timeout=None,
            max_retries=2,
        )

    def process_document(self, document: str) -> Tuple[List[Document], List[Document]]:
        """
        Process a document by splitting it into chunks and generating context for each chunk.
        """
        chunks = self.text_splitter.create_documents([document])
        contextualized_chunks = self._generate_contextualized_chunks(document, chunks)
        return chunks, contextualized_chunks

    def _generate_contextualized_chunks(self, document: str, chunks: List[Document]) -> List[Document]:
        """
        Generate contextualized versions of the given chunks.
        """
        contextualized_chunks = []
        for chunk in chunks:
            context = self._generate_context(document, chunk.page_content)
            contextualized_content = f"{context}\n\n{chunk.page_content}"
            contextualized_chunks.append(Document(page_content=contextualized_content, metadata=chunk.metadata))
        return contextualized_chunks

    def _generate_context(self, document: str, chunk: str) -> str:
        """
        Generate context for a specific chunk using the language model.
        """
        prompt = ChatPromptTemplate.from_template("""
        You are an AI assistant specializing in financial analysis, particularly for Tesla, Inc. Your task is to provide brief, relevant context for a chunk of text from Tesla's Q3 2023 financial report.
        Here is the financial report:
        <document>
        {document}
        </document>

        Here is the chunk we want to situate within the whole document::
        <chunk>
        {chunk}
        </chunk>

        Provide a concise context (2-3 sentences) for this chunk, considering the following guidelines:
        1. Identify the main financial topic or metric discussed (e.g., revenue, profitability, segment performance, market position).
        2. Mention any relevant time periods or comparisons (e.g., Q3 2023, year-over-year changes).
        3. If applicable, note how this information relates to Tesla's overall financial health, strategy, or market position.
        4. Include any key figures or percentages that provide important context.
        5. Do not use phrases like "This chunk discusses" or "This section provides". Instead, directly state the context.

        Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else.

        Context:
        """)
        messages = prompt.format_messages(document=document, chunk=chunk)
        response = self.llm.invoke(messages)
        return response.content

    def create_vectorstores(self, chunks: List[Document]) -> FAISS:
        """
        Create a vector store for the given chunks.
        """
        return FAISS.from_documents(chunks, self.embeddings)

    def create_bm25_index(self, chunks: List[Document]) -> BM25Okapi:
        """
        Create a BM25 index for the given chunks.
        """
        tokenized_chunks = [chunk.page_content.split() for chunk in chunks]
        return BM25Okapi(tokenized_chunks)

    @staticmethod
    def generate_cache_key(document: str) -> str:
        """
        Generate a cache key for a document.
        """
        return hashlib.md5(document.encode()).hexdigest()

    def generate_answer(self, query: str, relevant_chunks: List[str]) -> str:
        prompt = ChatPromptTemplate.from_template("""
        Based on the following information, please provide a concise and accurate answer to the question.
        If the information is not sufficient to answer the question, say so.

        Question: {query}

        Relevant information:
        {chunks}

        Answer:
        """)
        messages = prompt.format_messages(query=query, chunks="\n\n".join(relevant_chunks))
        response = self.llm.invoke(messages)
        return response.content

# Example financial document (approximately 5000 words, keyword-rich)
document = """
    Tesla, Inc. (TSLA) Financial Analysis and Market Overview - Q3 2023

    Executive Summary:
    Tesla, Inc. (NASDAQ: TSLA) continues to lead the electric vehicle (EV) market, showcasing strong financial performance and strategic growth initiatives in Q3 2023. This comprehensive analysis delves into Tesla's financial statements, market position, and future outlook, providing investors and stakeholders with crucial insights into the company's performance and potential.

    1. Financial Performance Overview:

    Revenue:
    Tesla reported total revenue of $23.35 billion in Q3 2023, marking a 9% increase year-over-year (YoY) from $21.45 billion in Q3 2022. The automotive segment remained the primary revenue driver, contributing $19.63 billion, up 5% YoY. Energy generation and storage revenue saw significant growth, reaching $1.56 billion, a 40% increase YoY.

    Profitability:
    Gross profit for Q3 2023 stood at $4.18 billion, with a gross margin of 17.9%. While this represents a decrease from the 25.1% gross margin in Q3 2022, it remains above industry averages. Operating income was $1.76 billion, resulting in an operating margin of 7.6%. Net income attributable to common stockholders was $1.85 billion, translating to diluted earnings per share (EPS) of $0.53.

    Cash Flow and Liquidity:
    Tesla's cash and cash equivalents at the end of Q3 2023 were $26.08 billion, a robust position that provides ample liquidity for ongoing operations and future investments. Free cash flow for the quarter was $0.85 billion, reflecting the company's ability to generate cash despite significant capital expenditures.

    2. Operational Highlights:

    Production and Deliveries:
    Tesla produced 430,488 vehicles in Q3 2023, a 17% increase YoY. The Model 3/Y accounted for 419,666 units, while the Model S/X contributed 10,822 units. Total deliveries reached 435,059 vehicles, up 27% YoY, demonstrating strong demand and improved production efficiency.

    Manufacturing Capacity:
    The company's installed annual vehicle production capacity increased to over 2 million units across its factories in Fremont, Shanghai, Berlin-Brandenburg, and Texas. The Shanghai Gigafactory remains the highest-volume plant, with an annual capacity exceeding 950,000 units.

    Energy Business:
    Tesla's energy storage deployments grew by 90% YoY, reaching 4.0 GWh in Q3 2023. Solar deployments also increased by 48% YoY to 106 MW, reflecting growing demand for Tesla's energy products.

    3. Market Position and Competitive Landscape:

    Global EV Market Share:
    Tesla maintained its position as the world's largest EV manufacturer by volume, with an estimated global market share of 18% in Q3 2023. However, competition is intensifying, particularly from Chinese manufacturers like BYD and established automakers accelerating their EV strategies.

    Brand Strength:
    Tesla's brand value continues to grow, ranked as the 12th most valuable brand globally by Interbrand in 2023, with an estimated brand value of $56.3 billion, up 4% from 2022.

    Technology Leadership:
    The company's focus on innovation, particularly in battery technology and autonomous driving capabilities, remains a key differentiator. Tesla's Full Self-Driving (FSD) beta program has expanded to over 800,000 customers in North America, showcasing its advanced driver assistance systems.

    4. Strategic Initiatives and Future Outlook:

    Product Roadmap:
    Tesla reaffirmed its commitment to launching the Cybertruck in 2023, with initial deliveries expected in Q4. The company also hinted at progress on a next-generation vehicle platform, aimed at significantly reducing production costs.

    Expansion Plans:
    Plans for a new Gigafactory in Mexico are progressing, with production expected to commence in 2025. This facility will focus on producing Tesla's next-generation vehicles and expand the company's North American manufacturing footprint.

    Battery Production:
    Tesla continues to ramp up its in-house battery cell production, with 4680 cells now being used in Model Y vehicles produced at the Texas Gigafactory. The company aims to achieve an annual production rate of 1,000 GWh by 2030.

    5. Risk Factors and Challenges:

    Supply Chain Constraints:
    While easing compared to previous years, supply chain issues continue to pose challenges, particularly in sourcing semiconductor chips and raw materials for batteries.

    Regulatory Environment:
    Evolving regulations around EVs, autonomous driving, and data privacy across different markets could impact Tesla's operations and expansion plans.

    Macroeconomic Factors:
    Rising interest rates and inflationary pressures may affect consumer demand for EVs and impact Tesla's profit margins.

    Competition:
    Intensifying competition in the EV market, especially in key markets like China and Europe, could pressure Tesla's market share and pricing power.

    6. Financial Ratios and Metrics:

    Profitability Ratios:
    - Return on Equity (ROE): 18.2%
    - Return on Assets (ROA): 10.3%
    - EBITDA Margin: 15.7%

    Liquidity Ratios:
    - Current Ratio: 1.73
    - Quick Ratio: 1.25

    Efficiency Ratios:
    - Asset Turnover Ratio: 0.88
    - Inventory Turnover Ratio: 11.2

    Valuation Metrics:
    - Price-to-Earnings (P/E) Ratio: 70.5
    - Price-to-Sales (P/S) Ratio: 7.8
    - Enterprise Value to EBITDA (EV/EBITDA): 41.2

    7. Segment Analysis:

    Automotive Segment:
    - Revenue: $19.63 billion (84% of total revenue)
    - Gross Margin: 18.9%
    - Key Products: Model 3, Model Y, Model S, Model X

    Energy Generation and Storage:
    - Revenue: $1.56 billion (7% of total revenue)
    - Gross Margin: 14.2%
    - Key Products: Powerwall, Powerpack, Megapack, Solar Roof

    Services and Other:
    - Revenue: $2.16 billion (9% of total revenue)
    - Gross Margin: 5.3%
    - Includes vehicle maintenance, repair, and used vehicle sales

    8. Geographic Revenue Distribution:

    - United States: $12.34 billion (53% of total revenue)
    - China: $4.67 billion (20% of total revenue)
    - Europe: $3.97 billion (17% of total revenue)
    - Other: $2.37 billion (10% of total revenue)

    9. Research and Development:

    Tesla invested $1.16 billion in R&D during Q3 2023, representing 5% of total revenue. Key focus areas include:
    - Next-generation vehicle platform development
    - Advancements in battery technology and production processes
    - Enhancements to Full Self-Driving (FSD) capabilities
    - Energy storage and solar technology improvements

    10. Capital Expenditures and Investments:

    Capital expenditures for Q3 2023 totaled $2.46 billion, primarily allocated to:
    - Expansion and upgrades of production facilities
    - Tooling for new products, including the Cybertruck
    - Supercharger network expansion
    - Investments in battery cell production capacity

    11. Debt and Capital Structure:

    As of September 30, 2023:
    - Total Debt: $5.62 billion
    - Total Equity: $43.51 billion
    - Debt-to-Equity Ratio: 0.13
    - Weighted Average Cost of Capital (WACC): 8.7%

    12. Stock Performance and Shareholder Returns:

    - 52-Week Price Range: $152.37 - $299.29
    - Market Capitalization: $792.5 billion (as of October 31, 2023)
    - Dividend Policy: Tesla does not currently pay dividends, reinvesting profits into growth initiatives
    - Share Repurchases: No significant share repurchases in Q3 2023

    13. Corporate Governance and Sustainability:

    Board Composition:
    Tesla's Board of Directors consists of 8 members, with 6 independent directors. The roles of CEO and Chairman are separate, with Robyn Denholm serving as Chairwoman.

    ESG Initiatives:
    - Environmental: Committed to using 100% renewable energy in all operations by 2030
    - Social: Focus on diversity and inclusion, with women representing 29% of the global workforce
    - Governance: Enhanced transparency in supply chain management and ethical sourcing of materials

    14. Analyst Recommendations and Price Targets:

    As of October 31, 2023:
    - Buy: 22 analysts
    - Hold: 15 analysts
    - Sell: 5 analysts
    - Average 12-month price target: $245.67

    15. Upcoming Catalysts and Events:

    - Cybertruck production ramp-up and initial deliveries (Q4 2023)
    - Investor Day 2024 (Date TBA)
    - Potential unveiling of next-generation vehicle platform (2024)
    - Expansion of FSD beta program to additional markets

    Conclusion:
    Tesla's Q3 2023 financial results demonstrate the company's continued leadership in the EV market, with strong revenue growth and operational improvements. While facing increased competition and margin pressures, Tesla's robust balance sheet, technological innovations, and expanding product portfolio position it well for future growth. Investors should monitor key metrics such as production ramp-up, margin trends, and progress on strategic initiatives to assess Tesla's long-term value proposition in the rapidly evolving automotive and energy markets.
    """

# Initialize ContextualRetrieval
cr = ContextualRetrieval()

cr

# Process the document
original_chunks, contextualized_chunks = cr.process_document(document)

len(contextualized_chunks)

print(original_chunks[0])

print(contextualized_chunks[0])

print(original_chunks[10])

print(contextualized_chunks[10])

# Create vectorstores
original_vectorstore = cr.create_vectorstores(original_chunks)
contextualized_vectorstore = cr.create_vectorstores(contextualized_chunks)

# Create BM25 indexes
original_bm25_index = cr.create_bm25_index(original_chunks)
contextualized_bm25_index = cr.create_bm25_index(contextualized_chunks)

# Generate cache key for the document
cache_key = cr.generate_cache_key(document)

cache_key

print(f"Processed {len(original_chunks)} chunks")
print(f"Cache key for the document: {cache_key}")

# Example queries related to financial information
queries = [
        "What was Tesla's total revenue in Q3 2023? what was the gross profit and cash position?",
        "How does the automotive gross margin in Q3 2023 compare to the previous year?",
        "What is Tesla's current debt-to-equity ratio?",
        "How much did Tesla invest in R&D during Q3 2023?",
        "What is Tesla's market share in the global EV market for Q3 2023?"
    ]



for query in queries:
        print(f"\nQuery: {query}")

        # Retrieve from original vectorstore
        original_vector_results = original_vectorstore.similarity_search(query, k=3)

        # Retrieve from contextualized vectorstore
        contextualized_vector_results = contextualized_vectorstore.similarity_search(query, k=3)

        # Retrieve from original BM25
        original_tokenized_query = query.split()
        original_bm25_results = original_bm25_index.get_top_n(original_tokenized_query, original_chunks, n=3)

        # Retrieve from contextualized BM25
        contextualized_tokenized_query = query.split()
        contextualized_bm25_results = contextualized_bm25_index.get_top_n(contextualized_tokenized_query, contextualized_chunks, n=3)

        # Generate answers
        original_vector_answer = cr.generate_answer(query, [doc.page_content for doc in original_vector_results])
        contextualized_vector_answer = cr.generate_answer(query, [doc.page_content for doc in contextualized_vector_results])
        original_bm25_answer = cr.generate_answer(query, [doc.page_content for doc in original_bm25_results])
        contextualized_bm25_answer = cr.generate_answer(query, [doc.page_content for doc in contextualized_bm25_results])


        print("\nOriginal Vector Search Results:")
        for i, doc in enumerate(original_vector_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nOriginal Vector Search Answer:")
        print(original_vector_answer)
        print("\n" + "-"*50)

        print("\nContextualized Vector Search Results:")
        for i, doc in enumerate(contextualized_vector_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nContextualized Vector Search Answer:")
        print(contextualized_vector_answer)
        print("\n" + "-"*50)

        print("\nOriginal BM25 Search Results:")
        for i, doc in enumerate(original_bm25_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nOriginal BM25 Search Answer:")
        print(original_bm25_answer)
        print("\n" + "-"*50)

        print("\nContextualized BM25 Search Results:")
        for i, doc in enumerate(contextualized_bm25_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nContextualized BM25 Search Answer:")
        print(contextualized_bm25_answer)

        print("\n" + "="*50)

# Complex queries requiring contextual information
    queries = [
        "How do Tesla's financial results in Q3 2023 reflect its overall strategy in both the automotive and energy sectors? Consider revenue growth, profitability, and investments in each sector.",

        "Analyze the relationship between Tesla's R&D spending, capital expenditures, and its financial performance. How might this impact its competitive position in the EV and energy storage markets over the next 3-5 years?",

        "Compare Tesla's financial health and market position in different geographic regions. How do regional variations in revenue, market share, and growth rates inform Tesla's global strategy?",

        "Evaluate Tesla's progress in vertical integration, considering its investments in battery production, software development, and manufacturing capabilities. How is this reflected in its financial statements and future outlook?",

        "Assess the potential impact of Tesla's Full Self-Driving (FSD) technology on its financial projections. Consider revenue streams, liability risks, and required investments in the context of the broader autonomous vehicle market.",

        "How does Tesla's financial performance and strategy in the energy storage and generation segment align with or diverge from its automotive business? What synergies or conflicts exist between these segments?",

        "Analyze Tesla's capital structure and liquidity position in the context of its growth strategy and market conditions. How well-positioned is the company to weather potential economic downturns or increased competition?",

        "Evaluate Tesla's pricing strategy across its product lines and geographic markets. How does this strategy impact its financial metrics, market share, and competitive positioning?",

        "Considering Tesla's current financial position, market trends, and competitive landscape, what are the most significant opportunities and risks for the company in the next 2-3 years? How might these factors affect its financial projections?",

        "Assess the potential financial implications of Tesla's expansion into new markets or product categories (e.g., Cybertruck, robotaxis, AI). How do these initiatives align with the company's core competencies and financial strategy?"
    ]

for query in queries:
        print(f"\nQuery: {query}")

        # Retrieve from original vectorstore
        original_vector_results = original_vectorstore.similarity_search(query, k=3)

        # Retrieve from contextualized vectorstore
        contextualized_vector_results = contextualized_vectorstore.similarity_search(query, k=3)

        # Retrieve from original BM25
        original_tokenized_query = query.split()
        original_bm25_results = original_bm25_index.get_top_n(original_tokenized_query, original_chunks, n=3)

        # Retrieve from contextualized BM25
        contextualized_tokenized_query = query.split()
        contextualized_bm25_results = contextualized_bm25_index.get_top_n(contextualized_tokenized_query, contextualized_chunks, n=3)

        # Generate answers
        original_vector_answer = cr.generate_answer(query, [doc.page_content for doc in original_vector_results])
        contextualized_vector_answer = cr.generate_answer(query, [doc.page_content for doc in contextualized_vector_results])
        original_bm25_answer = cr.generate_answer(query, [doc.page_content for doc in original_bm25_results])
        contextualized_bm25_answer = cr.generate_answer(query, [doc.page_content for doc in contextualized_bm25_results])


        print("\nOriginal Vector Search Results:")
        for i, doc in enumerate(original_vector_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nOriginal Vector Search Answer:")
        print(original_vector_answer)
        print("\n" + "-"*50)

        print("\nContextualized Vector Search Results:")
        for i, doc in enumerate(contextualized_vector_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nContextualized Vector Search Answer:")
        print(contextualized_vector_answer)
        print("\n" + "-"*50)

        print("\nOriginal BM25 Search Results:")
        for i, doc in enumerate(original_bm25_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nOriginal BM25 Search Answer:")
        print(original_bm25_answer)
        print("\n" + "-"*50)

        print("\nContextualized BM25 Search Results:")
        for i, doc in enumerate(contextualized_bm25_results, 1):
            print(f"{i}. {doc.page_content[:200]}...")

        print("\nContextualized BM25 Search Answer:")
        print(contextualized_bm25_answer)

        print("\n" + "="*50)




here is my code:
import os
import json
import shutil
import gradio as gr
from itertools import chain
from langchain_community.embeddings.sentence_transformer import SentenceTransformerEmbeddings
from langchain_unstructured import UnstructuredLoader
from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
from langchain_text_splitters import CharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_community.retrievers import ArxivRetriever
from langchain_community.tools import DuckDuckGoSearchRun
from langchain_community.document_loaders import DirectoryLoader
from langchain_ollama import ChatOllama

# For LangSmith tracing
from langsmith import traceable
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_API_KEY"] = "<TRUFFLEHOG_REDACTED_LANGSMITH>"  # Replace with your actual API key
os.environ["LANGCHAIN_PROJECT"] = "l"  # Optional: "default" is used if not set

# Initialize embeddings and vector store
embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
persist_directory = "./chroma_db"
vectorstore = Chroma(persist_directory=persist_directory, embedding_function=embeddings)

# Ensure the 'uploads' directory exists
if not os.path.exists("uploads"):
    os.makedirs("uploads")

# Function to load or initialize knowledge base JSON
def load_knowledge_base_json():
    if os.path.exists("knowledge_base.json"):
        with open("knowledge_base.json", "r") as f:
            knowledge_base_data = json.load(f)
    else:
        knowledge_base_data = {"files": [], "urls": []}
    return knowledge_base_data

knowledge_base_data = load_knowledge_base_json()

# Function to save knowledge base JSON
def save_knowledge_base_json():
    with open("knowledge_base.json", "w") as f:
        json.dump(knowledge_base_data, f)

# Function to get processed files
def get_processed_files():
    files = os.listdir("uploads")
    return "### Processed Files:\n" + "\n".join(files) if files else "No files processed yet."

# Function to display processed files and URLs
def display_processed_items():
    global knowledge_base_data
    files_text = get_processed_files()
    urls = knowledge_base_data.get("urls", [])
    display_text = files_text + "\n\n### Processed URLs:\n"
    if urls:
        display_text += "\n".join(urls)
    else:
        display_text += "No URLs processed yet."
    return display_text

# Function to update the knowledge base
@traceable
def update_knowledge_base(uploaded_files, website_urls):
    global knowledge_base_data
    if not uploaded_files and not website_urls:
        return "No new documents or URLs to process.", display_processed_items()

    processed_files = []
    skipped_files = []
    docs = []
    new_urls = []

    # Process uploaded files
    if uploaded_files:
        for file in uploaded_files:
            file_path = file.name
            destination_path = os.path.join("uploads", os.path.basename(file_path))
            file_name = os.path.basename(file_path)

            if os.path.exists(destination_path) or file_name in knowledge_base_data["files"]:
                skipped_files.append(file_name)
            else:
                try:
                    shutil.move(file_path, destination_path)
                    processed_files.append(file_name)
                except Exception as e:
                    print(f"Error moving file {file_name}: {str(e)}")

    # Load documents from uploaded files
    if processed_files:
        loader = DirectoryLoader("uploads", glob="**/*")
        documents = loader.load()
        docs.extend(documents)

    # Load documents from URLs
    if website_urls:
        urls = website_urls.split(',')
        for url in urls:
            url = url.strip()
            if url and url not in knowledge_base_data["urls"]:
                loader = UnstructuredLoader(web_url=url)
                try:
                    loaded_docs = loader.load()
                    docs.extend(loaded_docs)
                    new_urls.append(url)
                except Exception as e:
                    print(f"Error loading URL {url}: {str(e)}")
            else:
                skipped_files.append(url)

    # Split documents and add to vector store
    if docs:
        text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200, length_function=len)
        split_docs = text_splitter.split_documents(docs)
        vectorstore.add_documents(split_docs)
        vectorstore.persist()
        # Update knowledge base data and save to JSON
        knowledge_base_data["files"].extend(processed_files)
        knowledge_base_data["urls"].extend(new_urls)
        save_knowledge_base_json()
        status_message = f"Processed {len(processed_files)} new files and {len(new_urls)} new URLs."
        if skipped_files:
            status_message += f"\nSkipped existing files/URLs: {', '.join(skipped_files)}"
    else:
        status_message = "No new documents or URLs were added."
    return status_message, display_processed_items()

# Function to clear database and uploaded files
def clear_database():
    global vectorstore, knowledge_base_data

    try:
        vectorstore._client._client.close()
    except AttributeError:
        pass

    shutil.rmtree("uploads", ignore_errors=True)
    os.makedirs("uploads", exist_ok=True)

    shutil.rmtree(persist_directory, ignore_errors=True)

    vectorstore = Chroma(persist_directory=persist_directory, embedding_function=embeddings)

    # Clear the knowledge base data
    knowledge_base_data = {"files": [], "urls": []}
    # Remove the JSON file
    if os.path.exists("knowledge_base.json"):
        os.remove("knowledge_base.json")

    return "Knowledge base and uploaded files cleared.", display_processed_items()

# Function to process uploaded files in the chat and include their content
def process_uploaded_files(files):
    content = ""
    if files:
        for file in files:
            if file.name.endswith((".pdf", ".txt", ".docx")):
                loader = UnstructuredLoader(file_path=file.name)
                try:
                    docs = loader.load()
                    content += "\n".join([doc.page_content for doc in docs])
                except Exception as e:
                    print(f"Error processing file {file.name}: {str(e)}")
    return content

# Function to count tokens
def count_tokens(text):
    return len(text.split())  # Simple token count; adjust with tokenizer if needed

# Initialize Llama model
llm = ChatOllama(
    model="llama3.1",
    temperature=0,
    base_url="http://localhost:11434",
    streaming=True
)

# Maximum token limit
MAX_TOKENS = 128000  # Example maximum token limit

# Chat with Llama (normal chat) function
@traceable
def chat_with_llama(message, history, uploaded_files):
    # Process uploaded files and include their content
    file_content = process_uploaded_files(uploaded_files)
    # Convert history to list of messages
    messages = [SystemMessage(content="You are a helpful assistant. The user is able to upload files that get processed as text for you. You always answer to the users question by using the context or your knowlage in the same language the user wants you to talk in.")]
    if history:
        for user_msg, assistant_msg in history:
            if user_msg:
                messages.append(HumanMessage(content=user_msg))
                if assistant_msg is not None:
                    messages.append(AIMessage(content=assistant_msg))
    # Append file content to user message if any
    if file_content:
        message += f"\n\nContent files Uploaded:\n{file_content}"
        message += "\n\nWith that knowlage answer to the question as best as possible: "
    messages.append(HumanMessage(content=message))
    # Append the Question at the end again
    # Check if the total tokens exceed the maximum allowed tokens
    total_tokens = sum(count_tokens(msg.content) for msg in messages)
    if total_tokens > MAX_TOKENS:
        # Truncate the history to fit within the token limit
        while total_tokens > MAX_TOKENS and len(messages) > 2:
            # Remove the oldest user-assistant message pair
            messages.pop(1)
            messages.pop(1)
            total_tokens = sum(count_tokens(msg.content) for msg in messages)
        if total_tokens > MAX_TOKENS:
            return "Error: The conversation exceeds the maximum token limit. Please start a new chat."
    # Stream the response from the model
    try:
        response_stream = llm.stream(messages)
        partial_response = ""
        for chunk in response_stream:
            partial_response += chunk.content
            yield partial_response
    except Exception as e:
        yield f"Error: {str(e)}"

# Function to handle knowledge base Q&A using similarity search
@traceable
def knowledge_base_chat(message, history):
    # Search in the vector store using similarity search
    results = vectorstore.similarity_search(message, k=3)
    if not results:
        yield "No relevant information found in the knowledge base."
        return
    context = "\n\n".join([doc.page_content for doc in results])
    # Build the conversation history
    messages = [SystemMessage(content="You are a helpful assistant that answers questions based on the provided context.")]
    if history:
        for user_msg, assistant_msg in history:
            if user_msg:
                messages.append(HumanMessage(content=user_msg))
                if assistant_msg is not None:
                    messages.append(AIMessage(content=assistant_msg))
    # Add the latest message with context
    prompt = f"Context:\n{context}\n\nQuestion:\n{message}\n\nAnswer:"
    messages.append(HumanMessage(content=prompt))
    # Stream the response from the model
    try:
        response_stream = llm.stream(messages)
        partial_response = ""
        for chunk in response_stream:
            partial_response += chunk.content
            # Use yield to stream the response
            yield partial_response
    except Exception as e:
        yield f"Error: {str(e)}"

# Function for the Internet-Enabled LLM
@traceable
def internet_access_llm(message, history, use_wikipedia):
    # Generate a query from the user input
    query = message
    # Search the internet
    search_tool = DuckDuckGoSearchRun()
    search_results = search_tool.invoke(query)
    # Optionally search Wikipedia
    if use_wikipedia:
        wikipedia_tool = DuckDuckGoSearchRun(backend="wikipedia")
        wiki_results = wikipedia_tool.invoke(query)
        search_results += "\n\n" + wiki_results
    # Build the conversation history
    messages = [SystemMessage(content="You get information from the internet and answer the user's question with that information. Provide an accurate and detailed answer to the user's question using the search results.")]
    if history:
        for user_msg, assistant_msg in history:
            if user_msg:
                messages.append(HumanMessage(content=user_msg))
                if assistant_msg is not None:
                    messages.append(AIMessage(content=assistant_msg))
    # Add the latest message with search results
    prompt = f"Search Results:\n{search_results}\n\nQuestion:\n{message}\n\nAnswer:"
    messages.append(HumanMessage(content=prompt))
    # Stream the response from the model
    try:
        response_stream = llm.stream(messages)
        partial_response = ""
        for chunk in response_stream:
            partial_response += chunk.content
            yield partial_response
    except Exception as e:
        yield f"Error: {str(e)}"

# Function to retrieve arXiv papers and update knowledge base
@traceable
def arxiv_research(query, max_papers):
    global knowledge_base_data
    retriever = ArxivRetriever(load_max_docs=int(max_papers), get_full_documents=True)
    docs = retriever.get_relevant_documents(query)
    # Add documents to the vector store
    if docs:
        text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200, length_function=len)
        split_docs = text_splitter.split_documents(docs)
        vectorstore.add_documents(split_docs)
        vectorstore.persist()
        # Update knowledge base data and save to JSON
        arxiv_titles = [doc.metadata.get('Title', 'Unknown Title') for doc in docs]
        knowledge_base_data["files"].extend(arxiv_titles)
        save_knowledge_base_json()
        status_message = f"Added {len(docs)} papers to the knowledge base."
    else:
        status_message = "No documents were added from arXiv."
    return status_message, display_processed_items()

# Function to handle arXiv Knowledge Base Q&A using similarity search
@traceable
def ask_arxiv_knowledge_base(message, history):
    # Search in the vector store using similarity search
    results = vectorstore.similarity_search(message, k=3)
    if not results:
        yield "No relevant information found in the arXiv knowledge base."
        return
    context = "\n\n".join([doc.page_content for doc in results])
    # Build the conversation history
    messages = [SystemMessage(content="You are a helpful assistant that answers questions based on arXiv research papers.")]
    if history:
        for user_msg, assistant_msg in history:
            if user_msg:
                messages.append(HumanMessage(content=user_msg))
                if assistant_msg is not None:
                    messages.append(AIMessage(content=assistant_msg))
    # Add the latest message with context
    prompt = f"Context:\n{context}\n\nQuestion:\n{message}\n\nAnswer:"
    messages.append(HumanMessage(content=prompt))
    # Stream the response from the model
    try:
        response_stream = llm.stream(messages)
        partial_response = ""
        for chunk in response_stream:
            partial_response += chunk.content
            yield partial_response
    except Exception as e:
        yield f"Error: {str(e)}"

# Gradio interface
with gr.Blocks() as demo:
    gr.Markdown("# Multi-Tab Gradio App with Llama and Similarity Search")

    # Chat with Llama Tab
    with gr.Tab("Chat with Llama"):
        gr.Markdown("## Chat Interface")
        with gr.Row():
            upload_files = gr.File(
                label="Upload PDFs, DOCX or TXT files",
                file_types=[".pdf", ".txt", ".docx"],
                file_count="multiple",
                visible=True
            )
        gr_chat = gr.ChatInterface(
            fn=chat_with_llama,
            title="Chat with Llama",
            fill_height=True,  # Use fill_height instead of height
            additional_inputs=[upload_files]
        )

    # Knowledge Base Tab
    with gr.Tab("Knowledge Base"):
        gr.Markdown("## Knowledge Base Management and Chat")
        with gr.Row():
            with gr.Column(scale=1):
                gr.Markdown("### Update Knowledge Base")
                kb_upload_files = gr.File(
                    label="Upload Files",
                    file_types=[".pdf", ".txt", ".docx"],
                    file_count="multiple"
                )
                kb_website_urls = gr.Textbox(
                    label="Website URLs",
                    placeholder="Enter comma-separated URLs"
                )
                kb_update_button = gr.Button("Update Knowledge Base")
                kb_clear_button = gr.Button("Clear Database")
                kb_update_status = gr.Markdown()
                kb_update_button.click(
                    update_knowledge_base,
                    inputs=[kb_upload_files, kb_website_urls],
                    outputs=[kb_update_status, kb_update_status]
                )
                kb_clear_button.click(
                    clear_database,
                    outputs=[kb_update_status, kb_update_status]
                )
                gr.Markdown("### Processed Files and URLs")
                processed_items_display = gr.Markdown(display_processed_items())
                # Update the display when the knowledge base is updated or cleared
                kb_update_button.click(
                    update_knowledge_base,
                    inputs=[kb_upload_files, kb_website_urls],
                    outputs=[kb_update_status, processed_items_display]
                )
                kb_clear_button.click(
                    clear_database,
                    outputs=[kb_update_status, processed_items_display]
                )
            with gr.Column(scale=2):
                gr.Markdown("### Chat with Knowledge Base")
                gr_chat_kb = gr.ChatInterface(
                    fn=knowledge_base_chat,
                    title="Knowledge Base Chat",
                    fill_height=True  # Use fill_height instead of height
                )

    # LLM with Internet Access Tab
    with gr.Tab("LLM with Internet Access"):
        gr.Markdown("## Internet-Enabled LLM")
        use_wikipedia = gr.Checkbox(label="Use Wikipedia")
        gr_chat_internet = gr.ChatInterface(
            fn=internet_access_llm,
            title="LLM with Internet Access",
            additional_inputs=[use_wikipedia],
            fill_height=True  # Use fill_height instead of height
        )

    # arXiv Research Papers Tab
    with gr.Tab("arXiv Research Papers"):
        gr.Markdown("## Retrieve arXiv Papers and Chat")
        with gr.Row():
            with gr.Column(scale=1):
                gr.Markdown("### Retrieve arXiv Papers")
                arxiv_query = gr.Textbox(
                    label="Search Query",
                    placeholder="Enter your search query for arXiv papers"
                )
                max_papers = gr.Slider(
                    label="Number of Papers",
                    minimum=1,
                    maximum=10,
                    step=1,
                    value=5
                )
                arxiv_button = gr.Button("Retrieve Papers")
                arxiv_result = gr.Markdown()
                arxiv_button.click(
                    arxiv_research,
                    inputs=[arxiv_query, max_papers],
                    outputs=[arxiv_result, arxiv_result]
                )
                gr.Markdown("### Processed Files and URLs")
                processed_items_display_arxiv = gr.Markdown(display_processed_items())
                # Update the display when new papers are added
                arxiv_button.click(
                    arxiv_research,
                    inputs=[arxiv_query, max_papers],
                    outputs=[arxiv_result, processed_items_display_arxiv]
                )
            with gr.Column(scale=2):
                gr.Markdown("### Chat with arXiv Knowledge Base")
                gr_chat_arxiv = gr.ChatInterface(
                    fn=ask_arxiv_knowledge_base,
                    title="arXiv Knowledge Base Chat",
                    fill_height=True  # Use fill_height instead of height
                )

    # Apply CSS styling to adjust chatbox sizes if needed
    # Uncomment and adjust the CSS if 'fill_height' doesn't work as expected
    """
    demo.css = '''
    .gradio-chatbot {
        height: 600px !important;
    }
    '''
    """

    # Launch the app
    demo.launch()