Auf einer großen Knowledge Base mit RAG zusätzliche Informationen zur aktuellen Konversation finden. schreibe mir den code der das macht du kanst dich an folgenden code halten und möglichst viele elemente übernehmen. die knowledge base bitte selber anlegen und eher klein halten auch den corpus kannst du verwenden

import numpy as np
from openai import OpenAI  # Make sure you have the OpenAI library installed
import tiktoken  # Ensure you have tiktoken installed
import os

# --------------------------- Setup ---------------------------

# Connect to the local LM Studio server using OpenAI client.
# Replace the base_url and api_key with your actual server details.
client = OpenAI(base_url="http://localhost:1234/v1", api_key="lm-studio")

# Define your model names based on your LM Studio setup.
# Replace these with the actual names of your models.
# I dont need toset it looks like LM Studio is using the right model by default.
EMBEDDING_MODEL = "embedding-model-name"  # e.g., "text-embedding-ada-002"
CHAT_COMPLETION_MODEL = "chat-completion-model-name"  # e.g., "gpt-3.5-turbo"


# --------------------------- Step 1: Load Data ---------------------------

def load_corpus(file_path):
    """
    Loads text data from a specified file.

    :param file_path: Path to the text file.
    :return: Content of the file as a string.
    """
    print(f"Loading corpus from '{file_path}'...")
    try:
        with open(file_path, 'r', encoding='utf-8') as file:
            text = file.read()
            print("Corpus loaded successfully.")
            return text
    except FileNotFoundError:
        print(f"Error: The file '{file_path}' was not found.")
        return None
    except Exception as e:
        print(f"An error occurred while reading the file: {e}")
        return None


# --------------------------- Step 2: Split Text into Chunks ---------------------------

def split_into_chunks(text, chunk_size=50, overlap=10):
    """
    TODO: Find the perfect chunk size and overlap. 50 and 10 work for this example
    TODO: Explore the idea of variable chunk sizes and overlaps.
    Splits the text into smaller chunks. 50 might be a bit small many sentences are too short. Maybe even a varying
    length would be ideal so longer and shorter sentences can be found. Then the Chunks need to be filtered correctly.

    :param text: The complete text to split.
    :param chunk_size: Number of tokens per chunk.
    :param overlap: Number of overlapping tokens between chunks.
    :return: List of text chunks.
    """
    print("Splitting text into smaller chunks...")
    # TODO implement a better way to split the text into chunks via tokens seems not so efficient
    # tiktoken is a fast, efficient tokenizer designed specifically for OpenAI's models, such as GPT-3.5 and GPT-4.
    # It uses Byte Pair Encoding (BPE) tailored to how OpenAI models tokenize input data.
    # one could also use something provided by lm studio but this works als well
    encoding = tiktoken.get_encoding("cl100k_base")  # Encoding used by the model
    tokens = encoding.encode(text)  # Convert text to tokens
    chunks = []

    for i in range(0, len(tokens), chunk_size - overlap):
        chunk = tokens[i:i + chunk_size]  # Get a chunk of tokens
        decoded_chunk = encoding.decode(chunk)  # Convert tokens back to text
        chunks.append(decoded_chunk)
        print(f"Created chunk {len(chunks)}.")

    print(f"Total chunks created: {len(chunks)}")
    return chunks


# --------------------------- Step 3: Get Embeddings ---------------------------

def get_embeddings(texts):
    """
    Retrieves embeddings for a list of texts using the embedding model.

    :param texts: List of text strings.
    :return: List of embedding vectors.
    """
    print("Generating embeddings for text chunks...")
    try:
        response = client.embeddings.create(
            model=EMBEDDING_MODEL,  # Use your actual embeddings model
            input=texts
        )
        print("Embeddings generated successfully.")
        return [embedding.embedding for embedding in response.data]
    except Exception as e:
        print(f"An error occurred while getting embeddings: {e}")
        return None


# --------------------------- Step 4: Calculate Cosine Similarity ---------------------------

def cosine_similarity(vec1, vec2):
    """
    Calculates the cosine similarity between two vectors.

    :param vec1: First vector.
    :param vec2: Second vector.
    :return: Cosine similarity score.
    """
    vec1 = np.array(vec1)
    vec2 = np.array(vec2)
    norm1 = np.linalg.norm(vec1)
    norm2 = np.linalg.norm(vec2)
    if norm1 == 0 or norm2 == 0:
        return 0.0
    return np.dot(vec1, vec2) / (norm1 * norm2)


# --------------------------- Step 5: Retrieve Relevant Chunks ---------------------------

def retrieve_relevant_chunks(query, chunks, chunk_embeddings, top_n=10):
    """
    Finds the most relevant text chunks related to the query.

    :param query: User's query string.
    :param chunks: List of text chunks.
    :param chunk_embeddings: List of embeddings for each chunk.
    :param top_n: Number of top relevant chunks to retrieve.
    :return: List of tuples containing similarity score and chunk text.
    """
    print(f"\nRetrieving relevant chunks for the query: '{query}'")
    # Retrieve the embedding for the user query
    query_embedding = get_embeddings([query])  # Get embedding for the query
    if query_embedding is None:
        print("Failed to get query embedding.")
        return []

    # Extract the embedding from the response (assuming the response is a list of embeddings)
    query_embedding = query_embedding[0]  # Extract the embedding from the response

    # Initialize a list to store similarity scores and corresponding chunks
    scores = []

    # Calculate similarity between query and each chunk
    # The zip function pairs each element from the chunks list with the corresponding element from the
    # chunk_embeddings list.
    for chunk, chunk_embedding in zip(chunks, chunk_embeddings):
        # Calculate the cosine similarity between the query embedding and the chunk embedding
        score = cosine_similarity(query_embedding, chunk_embedding)
        # Append the similarity score and the chunk to the scores list this will help us find the text chunks that
        # have the hogest score and can be returned to the user
        scores.append((score, chunk))

    # Sort chunks by similarity score in descending order
    scores.sort(reverse=True, key=lambda x: x[0])

    # Select top_n chunks with positive similarity scores
    top_chunks = [(score, chunk) for score, chunk in scores[:top_n] if score > 0]

    print(f"Top {len(top_chunks)} relevant chunks retrieved.")
    for idx, (score, chunk) in enumerate(top_chunks, 1):
        print(f"{idx}. Score: {score:.4f}, Chunk: \"{chunk[:75]}...\"")  # Show first 75 chars
    return top_chunks


# --------------------------- Step 6: Check for Contradictions ---------------------------

def check_for_contradictions(user_statement, contextual_chunks):
    """
    Checks if the user's statement contradicts the provided context.

    :param user_statement: The statement to check.
    :param contextual_chunks: List of relevant text chunks.
    :return: Response from the chat model regarding contradiction.
    """
    print(f"\nChecking for contradictions against the statement: '{user_statement}'")
    context = "\n".join(chunk for _, chunk in contextual_chunks)  # Combine all chunks into context

    messages = [
        {"role": "system", "content": "You are an assistant that checks for contradictions."},
        {"role": "user",
         "content": f"Context: {context}\nUser Statement: {user_statement}\n\nDoes the user statement contradict the context? Answer yes or no and provide a brief explanation."}
    ]

    print("Sending contradiction check request to the chat model...")

    try:
        completion = client.chat.completions.create(
            model=CHAT_COMPLETION_MODEL,  # Use your actual chat completion model
            messages=messages,
            temperature=0.5,
        )
        response_message = completion.choices[0].message.content
        print("Contradiction check completed.")
        return response_message
    except Exception as e:
        print(f"An error occurred during contradiction check: {e}")
        return "Error occurred during contradiction check."


# --------------------------- Step 7: Calculate Word Importance ---------------------------

def word_importance_calculation(query, original_top_chunks, chunks, chunk_embeddings):
    """
    Determines the most important words in the user's query based on their impact on retrieved chunks.

    :param query: User's query string.
    :param original_top_chunks: Original top relevant chunks based on the full query.
    :param chunks: All text chunks.
    :param chunk_embeddings: Embeddings for all text chunks.
    :return: List of important words.
    """
    print("\nCalculating word importance in the query...")
    query_words = query.split()
    word_scores = {}

    for word in query_words:
        # Remove the current word from the query
        modified_query = " ".join(w for w in query_words if w.lower() != word.lower())
        if not modified_query:
            continue  # Skip if all words are removed

        # Retrieve chunks based on the modified query
        modified_top_chunks = retrieve_relevant_chunks(modified_query, chunks, chunk_embeddings, top_n=5)

        # Find common chunks between original and modified queries
        original_chunks_set = set(chunk for _, chunk in original_top_chunks)
        modified_chunks_set = set(chunk for _, chunk in modified_top_chunks)
        common_chunks = original_chunks_set.intersection(modified_chunks_set)

        # The importance score is the number of common chunks
        word_scores[word] = len(common_chunks)
        print(f"Word: '{word}', Common Chunks: {len(common_chunks)}")

    # Sort words by their importance score in descending order
    importance_words = sorted(word_scores, key=word_scores.get, reverse=True)[:5]
    print("Word importance calculation completed.")
    return importance_words


# --------------------------- Main Function ---------------------------

def main():
    """
    Main function to run the interactive console for Retrieval-Augmented Generation (RAG).
    """
    print("==============================================")
    print(" Welcome to the RAG Interactive Console!")
    print("==============================================")
    print("You can enter your queries to retrieve relevant information.")
    print("Type 'exit' to quit the program.\n")

    # Step 1: Load and preprocess the corpus
    corpus_file = 'corpus_text.txt'  # Replace with your actual corpus file path
    text = load_corpus(corpus_file)
    if text is None:
        print("Failed to load corpus. Exiting the program.")
        return

    # Step 2: Split the text into chunks
    chunks = split_into_chunks(text)

    # Step 3: Get embeddings for each chunk
    chunk_embeddings = get_embeddings(chunks)
    if chunk_embeddings is None:
        print("Failed to generate embeddings. Exiting the program.")
        return

    # Start interactive loop
    while True:
        # Get user query
        query = input("\nPlease enter your query (or type 'exit' to quit): ").strip()
        if query.lower() == 'exit':
            print("Exiting the program. Goodbye!")
            break
        if not query:
            print("Empty query received. Please enter a valid query.")
            continue

        # Retrieve relevant chunks based on the query
        top_chunks = retrieve_relevant_chunks(query, chunks, chunk_embeddings, top_n=10)

        if not top_chunks:
            print("No relevant information found for your query.")
            continue

        # Display top 3 relevant chunks
        print("\nTop 3 Relevant Information:")
        for idx, (score, chunk) in enumerate(top_chunks[:3], 1):
            print(f"{idx}. [Score: {score:.4f}] {chunk[:200]}...")  # Display first 200 characters

        # Ask user for a statement to check for contradictions
        user_statement = input("\nEnter a statement to check for contradictions (or press Enter to skip): ").strip()
        if user_statement:
            contradiction_response = check_for_contradictions(user_statement, top_chunks)
            print(f"\nContradiction Check Response:\n{contradiction_response}")
        else:
            print("No statement entered. Skipping contradiction check.")

        # Calculate and display the most important words in the query
        most_important_words = word_importance_calculation(query, top_chunks, chunks, chunk_embeddings)
        if most_important_words:
            print("\nMost Important Words in Your Query:")
            for word in most_important_words:
                print(f"- {word}")
        else:
            print("Could not determine important words in the query.")


# --------------------------- Entry Point ---------------------------

if __name__ == "__main__":
    main()