I have this code and it uses the ticktoken package

import numpy as np
from openai import OpenAI  # Make sure you have the OpenAI library installed
import tiktoken  # Ensure you have tiktoken installed
import os

# --------------------------- Setup ---------------------------

# Connect to the local LM Studio server using OpenAI client.
# Replace the base_url and api_key with your actual server details.
client = OpenAI(base_url="http://localhost:1234/v1", api_key="lm-studio")

# Define your model names based on your LM Studio setup.
# Replace these with the actual names of your models.
EMBEDDING_MODEL = "embedding-model-name"  # e.g., "text-embedding-ada-002"
CHAT_COMPLETION_MODEL = "chat-completion-model-name"  # e.g., "gpt-3.5-turbo"


# --------------------------- Step 1: Load Data ---------------------------

def load_corpus(file_path):
    """
    Loads text data from a specified file.

    :param file_path: Path to the text file.
    :return: Content of the file as a string.
    """
    print(f"Loading corpus from '{file_path}'...")
    try:
        with open(file_path, 'r', encoding='utf-8') as file:
            text = file.read()
            print("Corpus loaded successfully.")
            return text
    except FileNotFoundError:
        print(f"Error: The file '{file_path}' was not found.")
        return None
    except Exception as e:
        print(f"An error occurred while reading the file: {e}")
        return None


# --------------------------- Step 2: Split Text into Chunks ---------------------------

def split_into_chunks(text, chunk_size=50, overlap=10):
    """
    Splits the text into smaller chunks. 5o0 might be a bit small many sentences are too short. Maybe even a varying 
    length would be ideal so longer and shorter sentences can be found. Then the Chunks need to be filtered correctly.

    :param text: The complete text to split.
    :param chunk_size: Number of tokens per chunk.
    :param overlap: Number of overlapping tokens between chunks.
    :return: List of text chunks.
    """
    print("Splitting text into smaller chunks...")
    encoding = tiktoken.get_encoding("cl100k_base")  # Encoding used by the model
    tokens = encoding.encode(text)  # Convert text to tokens
    chunks = []

    for i in range(0, len(tokens), chunk_size - overlap):
        chunk = tokens[i:i + chunk_size]  # Get a chunk of tokens
        decoded_chunk = encoding.decode(chunk)  # Convert tokens back to text
        chunks.append(decoded_chunk)
        print(f"Created chunk {len(chunks)}.")

    print(f"Total chunks created: {len(chunks)}")
    return chunks


# --------------------------- Step 3: Get Embeddings ---------------------------

def get_embeddings(texts):
    """
    Retrieves embeddings for a list of texts using the embedding model.

    :param texts: List of text strings.
    :return: List of embedding vectors.
    """
    print("Generating embeddings for text chunks...")
    try:
        response = client.embeddings.create(
            model=EMBEDDING_MODEL,  # Use your actual embeddings model
            input=texts
        )
        print("Embeddings generated successfully.")
        return [embedding.embedding for embedding in response.data]
    except Exception as e:
        print(f"An error occurred while getting embeddings: {e}")
        return None


# --------------------------- Step 4: Calculate Cosine Similarity ---------------------------

def cosine_similarity(vec1, vec2):
    """
    Calculates the cosine similarity between two vectors.

    :param vec1: First vector.
    :param vec2: Second vector.
    :return: Cosine similarity score.
    """
    vec1 = np.array(vec1)
    vec2 = np.array(vec2)
    norm1 = np.linalg.norm(vec1)
    norm2 = np.linalg.norm(vec2)
    if norm1 == 0 or norm2 == 0:
        return 0.0
    return np.dot(vec1, vec2) / (norm1 * norm2)


# --------------------------- Step 5: Retrieve Relevant Chunks ---------------------------

def retrieve_relevant_chunks(query, chunks, chunk_embeddings, top_n=10):
    """
    Finds the most relevant text chunks related to the query.

    :param query: User's query string.
    :param chunks: List of text chunks.
    :param chunk_embeddings: List of embeddings for each chunk.
    :param top_n: Number of top relevant chunks to retrieve.
    :return: List of tuples containing similarity score and chunk text.
    """
    print(f"\nRetrieving relevant chunks for the query: '{query}'")
    query_embedding = get_embeddings([query])  # Get embedding for the query
    if query_embedding is None:
        print("Failed to get query embedding.")
        return []

    query_embedding = query_embedding[0]  # Extract the embedding from the response
    scores = []

    # Calculate similarity between query and each chunk
    for chunk, chunk_embedding in zip(chunks, chunk_embeddings):
        score = cosine_similarity(query_embedding, chunk_embedding)
        scores.append((score, chunk))

    # Sort chunks by similarity score in descending order
    scores.sort(reverse=True, key=lambda x: x[0])

    # Select top_n chunks with positive similarity scores
    top_chunks = [(score, chunk) for score, chunk in scores[:top_n] if score > 0]

    print(f"Top {len(top_chunks)} relevant chunks retrieved.")
    for idx, (score, chunk) in enumerate(top_chunks, 1):
        print(f"{idx}. Score: {score:.4f}, Chunk: \"{chunk[:75]}...\"")  # Show first 75 chars
    return top_chunks


# --------------------------- Step 6: Check for Contradictions ---------------------------

def check_for_contradictions(user_statement, contextual_chunks):
    """
    Checks if the user's statement contradicts the provided context.

    :param user_statement: The statement to check.
    :param contextual_chunks: List of relevant text chunks.
    :return: Response from the chat model regarding contradiction.
    """
    print(f"\nChecking for contradictions against the statement: '{user_statement}'")
    context = "\n".join(chunk for _, chunk in contextual_chunks)  # Combine all chunks into context

    messages = [
        {"role": "system", "content": "You are an assistant that checks for contradictions."},
        {"role": "user",
         "content": f"Context: {context}\nUser Statement: {user_statement}\n\nDoes the user statement contradict the context? Answer yes or no and provide a brief explanation."}
    ]

    print("Sending contradiction check request to the chat model...")

    try:
        completion = client.chat.completions.create(
            model=CHAT_COMPLETION_MODEL,  # Use your actual chat completion model
            messages=messages,
            temperature=0.5,
        )
        response_message = completion.choices[0].message.content
        print("Contradiction check completed.")
        return response_message
    except Exception as e:
        print(f"An error occurred during contradiction check: {e}")
        return "Error occurred during contradiction check."


# --------------------------- Step 7: Calculate Word Importance ---------------------------

def word_importance_calculation(query, original_top_chunks, chunks, chunk_embeddings):
    """
    Determines the most important words in the user's query based on their impact on retrieved chunks.

    :param query: User's query string.
    :param original_top_chunks: Original top relevant chunks based on the full query.
    :param chunks: All text chunks.
    :param chunk_embeddings: Embeddings for all text chunks.
    :return: List of important words.
    """
    print("\nCalculating word importance in the query...")
    query_words = query.split()
    word_scores = {}

    for word in query_words:
        # Remove the current word from the query
        modified_query = " ".join(w for w in query_words if w.lower() != word.lower())
        if not modified_query:
            continue  # Skip if all words are removed

        # Retrieve chunks based on the modified query
        modified_top_chunks = retrieve_relevant_chunks(modified_query, chunks, chunk_embeddings, top_n=5)

        # Find common chunks between original and modified queries
        original_chunks_set = set(chunk for _, chunk in original_top_chunks)
        modified_chunks_set = set(chunk for _, chunk in modified_top_chunks)
        common_chunks = original_chunks_set.intersection(modified_chunks_set)

        # The importance score is the number of common chunks
        word_scores[word] = len(common_chunks)
        print(f"Word: '{word}', Common Chunks: {len(common_chunks)}")

    # Sort words by their importance score in descending order
    importance_words = sorted(word_scores, key=word_scores.get, reverse=True)[:5]
    print("Word importance calculation completed.")
    return importance_words


# --------------------------- Main Function ---------------------------

def main():
    """
    Main function to run the interactive console for Retrieval-Augmented Generation (RAG).
    """
    print("==============================================")
    print(" Welcome to the RAG Interactive Console!")
    print("==============================================")
    print("You can enter your queries to retrieve relevant information.")
    print("Type 'exit' to quit the program.\n")

    # Step 1: Load and preprocess the corpus
    corpus_file = 'corpus_text.txt'  # Replace with your actual corpus file path
    text = load_corpus(corpus_file)
    if text is None:
        print("Failed to load corpus. Exiting the program.")
        return

    # Step 2: Split the text into chunks
    chunks = split_into_chunks(text)

    # Step 3: Get embeddings for each chunk
    chunk_embeddings = get_embeddings(chunks)
    if chunk_embeddings is None:
        print("Failed to generate embeddings. Exiting the program.")
        return

    # Start interactive loop
    while True:
        # Get user query
        query = input("\nPlease enter your query (or type 'exit' to quit): ").strip()
        if query.lower() == 'exit':
            print("Exiting the program. Goodbye!")
            break
        if not query:
            print("Empty query received. Please enter a valid query.")
            continue

        # Retrieve relevant chunks based on the query
        top_chunks = retrieve_relevant_chunks(query, chunks, chunk_embeddings, top_n=10)

        if not top_chunks:
            print("No relevant information found for your query.")
            continue

        # Display top 3 relevant chunks
        print("\nTop 3 Relevant Information:")
        for idx, (score, chunk) in enumerate(top_chunks[:3], 1):
            print(f"{idx}. [Score: {score:.4f}] {chunk[:200]}...")  # Display first 200 characters

        # Ask user for a statement to check for contradictions
        user_statement = input("\nEnter a statement to check for contradictions (or press Enter to skip): ").strip()
        if user_statement:
            contradiction_response = check_for_contradictions(user_statement, top_chunks)
            print(f"\nContradiction Check Response:\n{contradiction_response}")
        else:
            print("No statement entered. Skipping contradiction check.")

        # Calculate and display the most important words in the query
        most_important_words = word_importance_calculation(query, top_chunks, chunks, chunk_embeddings)
        if most_important_words:
            print("\nMost Important Words in Your Query:")
            for word in most_important_words:
                print(f"- {word}")
        else:
            print("Could not determine important words in the query.")


# --------------------------- Entry Point ---------------------------

if __name__ == "__main__":
    main()

but i have an lm studio embeding model available isnt that better since bifgger and wruns on gpu or is that different from what this libary offers

should i use my lm studio embedding model or not what are the dis adventages what would i gain from chnging the code to using the lm studio model?