Crea el código para maximizar la similitud semántica en el siguiente código con el Framework Ray Tune. El código fuente es el siguiente: "# -*- coding: utf-8 -*-
import os

"""Partition, and chunk your file so that the logical structure of the document is preserved for better RAG results."""

import fitz  # PyMuPDF

# Abre el documento PDF
archivo_pdf = "Full.pdf"
doc = fitz.open(archivo_pdf)

# Lista para contener ‘elements’ con texto y metadatos
elements = []

# Iterar a través de las páginas para extracción de texto
for pagina in doc:
    # Extraer texto de la página
    texto_pagina = pagina.get_text()

    # Extraer metadatos (esto será igual para todas las páginas, así que podrías hacerlo una vez fuera del bucle)
    metadata = doc.metadata

    # Añadir texto y metadatos a ‘elements’
    elements.append({"text": texto_pagina, "metadata": metadata})

"""Create LangChain documents from document chunks and their metadata, and ingest those documents into the FAISS vectorstore.

Set up the retriever.
"""

from langchain_core.documents import Document
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceEmbeddings

documents = []
for element in elements:
    # Si tu ‘element’ es un objeto con métodos, asegúrate de cambiar los accesos a atributos/propiedades adecuadamente
    metadata = element['metadata'] if 'metadata' in element else {}
    documents.append(Document(page_content=element['text'], metadata=metadata))

db = FAISS.from_documents(documents, HuggingFaceEmbeddings(model_name="BAAI/bge-base-en-v1.5"))
retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": 4})

"""To run this tutorial in the free Colab GPU, we'll need to quantize the model:"""

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig

model_name = "meta-llama/Meta-Llama-3-8B-Instruct"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=bnb_config)
tokenizer = AutoTokenizer.from_pretrained(model_name)

from langchain.llms import HuggingFacePipeline
from langchain.prompts import PromptTemplate
from transformers import pipeline
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

terminators = [
    tokenizer.eos_token_id,
    tokenizer.convert_tokens_to_ids("
")
]

text_generation_pipeline = pipeline(
    model=model,
    tokenizer=tokenizer,
    task="text-generation",
    temperature=0.2,
    do_sample=True,
    repetition_penalty=1.1,
    return_full_text=False,
    max_new_tokens=200,
    eos_token_id=terminators,
)

llm = HuggingFacePipeline(pipeline=text_generation_pipeline)

prompt_template = """

user

You are an assistant for answering questions about IPM.
You are given the extracted parts of a long document and a question. Provide a conversational answer.
If you don't know the answer, just say "I do not know." Don't make up an answer.
Question: {question}
Context: {context}

assistant

"""

prompt = PromptTemplate(
    input_variables=["context", "question"],
    template=prompt_template,
)

llm_chain = prompt | llm | StrOutputParser()
rag_chain = {"context": retriever, "question": RunnablePassthrough()} | llm_chain


"""## Integrado"""

import json
import gc
# Cargar el archivo JSON
with open('Full.json', 'r') as file:
    data = json.load(file)

from sentence_transformers import SentenceTransformer, util

# Initialize your sentence transformer model for semantic similarity calculation
model = SentenceTransformer('all-MiniLM-L6-v2')

# Obtener las respuestas del JSON
responses = list(data["responses"].values())
queries = list(data["queries"].values())

# Calcular la similitud del coseno
similarity_scores = []

print(queries)
for query in queries:
  gc.collect()
  answers = rag_chain.invoke(query)
  print(answers)
  gc.collect()

  # Encode both the given answer and the expected answer to get their embeddings
  given_answer_embedding = model.encode(answers, convert_to_tensor=True)
  expected_answer_embedding = model.encode(responses[queries.index(query)], convert_to_tensor=True)

  # Calculate cosine similarity
  similarity = util.cos_sim(given_answer_embedding, expected_answer_embedding)
"