Repository: explodinggradients/ragas
Issue: API key
Root cause: Validation Gap

[ ] I checked the [documentation](https://docs.ragas.io/) and related resources and couldn't find an answer to my question.

**Your Question**
I create a subclass from baseragasembeddins and I want to use evaluate function for some metrics. do I need to have access to openAI for API key or any thing else?(actuaclly the main thing is that I have embeddings and I do not want to pay again for openAI for embeddings,thus I want to be able to use other sources like mistral or ....)


**Code Examples**
from ragas.embeddings import BaseRagasEmbeddings
import numpy as np

class CustomListEmbeddings(BaseRagasEmbeddings):
    def __init__(self, precomputed_embeddings):
        """
        :param precomputed_embeddings: A list of precomputed embeddings.
        """
        self.precomputed_embeddings = precomputed_embeddings

    def get_embedding(self, index: str) -> np.ndarray:
        """
        Return the corresponding embedding for a given index.
        Here we assume the text input is just an index (or identifier) as a string.
        """
        # Convert the input string to an integer index to access the list of embeddings.
        index = int(index)  # Convert text (index) to an integer
        return np.array(self.precomputed_embeddings[index])

    def get_embeddings(self, indices: list) -> np.ndarray:
        """
        Return embeddings for a list of indices.
        """
        embeddings = [self.get_embedding(index) for index in indices]
        return np.vstack(embeddings)  # Stack them into a 2D array
**Additional context**
Anything else you want to share with us? 



<!-- Answer -->
You don't need an OpenAI API key to use the `evaluate` function with a subclass of `BaseRagasEmbeddings`. You can use other sources like Mistral for embeddings without additional costs, as long as they are subclasses of `BaseRagasEmbeddings` <sup>[[1]](https://github.com/explodinggradients/ragas/blob/main/docs/howtos/customisations/bring-your-own-llm-or-embs.md)</sup>.


<!-- Dosu Comment Footer -->
*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*


this is my code: 
import json
import pandas as pd
import numpy as np
from datasets import Dataset
from langchain.embeddings import HuggingFaceEmbeddings
from ragas.embeddings import LangchainEmbeddingsWrapper
from ragas.embeddings import BaseRagasEmbeddings
from ragas.metrics import faithfulness, context_utilization
from ragas import evaluate

# Load the ground truth data
file_path = 'assets\\GT.json'
with open(file_path) as f:
    ground_truth_data = json.load(f)

# Load the question and the answer and the chunks
file_path = 'assets\\user_llm_interaction_embeddings_c1521dd5_b819_4241_b3a4_3e5c1388037c.json'
with open(file_path) as f:
    llm = json.load(f)

# Initialize an empty list to hold the new dataset
data_set = []

# Iterate through the list and combine every two dictionaries
for i in range(0, len(llm), 2):
    combined_dict = {
        "text_vector_1": llm[i].get("text_vector", []),
        "text_vector_2": llm[i + 1].get("text_vector", []),
        'chunks': llm[i + 1].get('chunks', [])
    }
    data_set.append(combined_dict)

def map_chunks(data_set, ground_truth_data):
    for item in data_set:  # Iterate over each dictionary in data_set
        c = []  # Reset c for each item
        for chunk_id in item['chunks']:  # Loop through 'chunks' in the current dictionary
            for element in ground_truth_data:  # Loop through ground_truth_data
                if element['id'] == chunk_id:  # Match chunk_id with element's id
                    c.append(element['text_vector'])  # Append the matching text_vector to c
        item['chunks'] = c  # Replace the original 'chunks' (ids) with the mapped text_vector values

    return data_set  # Return the updated data_set

data_set = map_chunks(data_set, ground_truth_data)

# Create the `ragas_data` as a list of dictionaries
ragas_data = [
    {
        "question": entry["text_vector_1"],  # List of strings
        "answer": entry["text_vector_2"],    # List of strings
        "contexts": entry["chunks"]           # List of lists of strings
    }
    for entry in data_set
]

# Create the required structure for Dataset
formatted_data = {
    "question": [entry["question"] for entry in ragas_data],
    "contexts": [entry["contexts"] for entry in ragas_data],
    "answer": [entry["answer"] for entry in ragas_data]
}

# Convert to a Hugging Face Dataset
dataset = Dataset.from_dict(formatted_data)

# Verify the dataset structure
print(dataset)

# Creating a subclass which is compatible with BaseRagasEmbeddings
class CustomListEmbeddings(BaseRagasEmbeddings):
    def __init__(self, dataset):
        """
        :param dataset: A list of precomputed embeddings.
        """
        self.dataset = dataset

    def get_embedding(self, index: str) -> np.ndarray:
        """
        Return the corresponding embedding for a given index.
        Here we assume the input is just an index (or identifier) as a string.
        """
        index = int(index)  # Convert index to an integer
        return np.array(self.dataset[index])

    def get_embeddings(self, indices: list) -> np.ndarray:
        """
        Return embeddings for a list of indices.
        """
        embeddings = [self.get_embedding(index) for index in indices]
        return np.vstack(embeddings)  # Stack them into a 2D array

    def embed_documents(self, docs: list) -> np.ndarray:
        """
        Embeds a list of documents (represented as indices).
        """
        return self.get_embeddings(docs)

    def embed_query(self, query: str) -> np.ndarray:
        """
        Embeds a single query (represented as an index).
        """
        return self.get_embedding(query)

# Initialize your custom embeddings
custom_embeddings = CustomListEmbeddings(formatted_data["question"])

# Step 4: Define the list of evaluation metrics
metrics = [faithfulness, context_utilization]

# Step 5: Evaluate the data using the RAGAS `evaluate` function
evaluation_report = evaluate(dataset=dataset, metrics=metrics, embeddings=custom_embeddings)

# Step 6: Display the evaluation results
print("RAGAS Evaluation Report:")
print(evaluation_report)

# Optional: Convert the evaluation report to a DataFrame for better readability
evaluation_df = pd.DataFrame(evaluation_report.items(), columns=['Metric', 'Score'])
print("\nEvaluation Report as DataFrame:")
print(evaluation_df)

and I get the following error:
Exception has occurred: ValidationError
4608 validation errors for MultiTurnSample
user_input -> 0
  value is not a valid dict (type=type_error.dict)
user_input -> 0
  value is not a valid dict (type=type_error.dict)
user_input -> 0
  value is not a valid dict (type=type_error.dict)

<!-- Answer -->
The error you're encountering suggests that the dataset structure might not match the expected schema for the `evaluate` function. Here's how you can resolve it:

1. **Dataset Structure**: Ensure your dataset is structured correctly. The expected format is:

   - `question`: list of strings
   - `contexts`: list of lists of strings
   - `answer`: list of strings

2. **Validation**: The error indicates that the `user_input` is not a valid dictionary. Make sure each entry in your dataset is a dictionary with the correct keys and types.

3. **Column Mapping**: If your dataset columns do not match the expected names, use the `column_map` parameter to map your dataset's column names to the expected ones. For example:

   ```python
   column_map = {"your_question_column": "question", "your_contexts_column": "contexts", "your_answer_column": "answer"}
   ```

4. **Embedding Compatibility**: Ensure your custom embeddings are compatible with the expected `LlamaIndexEmbeddings`. If they are not, make sure they are properly wrapped.

Review these points and adjust your dataset and embeddings accordingly. If the issue persists, double-check the data being passed to ensure it adheres to the expected format <sup>[[1]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/evaluation.py)</sup><sup>[[2]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/integrations/llama_index.py)</sup>.


<!-- Dosu Comment Footer -->
*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*


I create a subclass of baseragasembedding but it still give me the error :
Exception has occurred: OpenAIError
The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable
  File "C:\Users\Amin\OneDrive - unige.it\Desktop\tirocini\code\version01rageva.py", line 97, in <module>
    evaluation_report = evaluate(dataset=ragas_data, metrics=metrics, embeddings=custom_embeddings)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable


also I put my code here so u can check and the metric that i used dont need api_key.
my code:
import json
import pandas as pd
import numpy as np
from datasets import Dataset
from ragas.embeddings import BaseRagasEmbeddings
from ragas.metrics import faithfulness, context_utilization
from ragas import evaluate

# Load the ground truth data
file_path = 'assets\\GT.json'
with open(file_path) as f:
    ground_truth_data = json.load(f)

# Load the question and the answer and the chunks
file_path = 'assets\\user_llm_interaction_embeddings_c1521dd5_b819_4241_b3a4_3e5c1388037c.json'
with open(file_path) as f:
    llm = json.load(f)

# Initialize an empty list to hold the new dataset
data_set = []

# Iterate through the list and combine every two dictionaries
for i in range(0, len(llm), 2):
    combined_dict = {
        "text_vector_1": llm[i].get("text_vector", []),
        "text_vector_2": llm[i + 1].get("text_vector", []),
        'chunks': llm[i + 1].get('chunks', [])
    }
    data_set.append(combined_dict)

def map_chunks(data_set, ground_truth_data):
    for item in data_set:  # Iterate over each dictionary in data_set
        c = []  # Reset c for each item
        for chunk_id in item['chunks']:  # Loop through 'chunks' in the current dictionary
            for element in ground_truth_data:  # Loop through ground_truth_data
                if element['id'] == chunk_id:  # Match chunk_id with element's id
                    c.append(element['text_vector'])  # Append the matching text_vector to c
        item['chunks'] = c  # Replace the original 'chunks' (ids) with the mapped text_vector values

    return data_set  # Return the updated data_set

data_set = map_chunks(data_set, ground_truth_data)

# Assuming data_set is a list of dictionaries
ragas_data = [
    {
        "question": entry["text_vector_1"],  # Assuming this is a list of strings
        "answer": entry["text_vector_2"],  # Assuming this is a list of strings
        "contexts": entry["chunks"]  # Assuming this is a list of lists of strings
    }
    for entry in data_set
]

# Create the required structure for Dataset
formatted_data = {
    "question": [entry["question"] for entry in ragas_data],
    "contexts": [entry["contexts"] for entry in ragas_data],
    "answer": [entry["answer"] for entry in ragas_data]
}

class CustomListEmbeddings(BaseRagasEmbeddings):
    def __init__(self, ragas_data):
        """
        :param ragas_data: A list of dictionaries containing precomputed embeddings.
        """
        self.ragas_data = ragas_data

    def get_embedding(self, index: int, key: str) -> np.ndarray:
        """
        Fetch the embedding for the specified key (question, answer, or contexts) from ragas_data.
        :param index: The index of the entry.
        :param key: The key (question, answer, contexts) to fetch the embedding.
        """
        return np.array(self.ragas_data[index][key])

    def embed_documents(self, indices: list) -> np.ndarray:
        """
        Fetch the embeddings for 'contexts' based on the indices.
        :param indices: A list of indices for the documents to embed.
        """
        return np.vstack([self.get_embedding(index, 'contexts') for index in indices])

    def embed_query(self, query_index: str) -> np.ndarray:
        """
        Fetch the embedding for 'question' based on the index.
        :param query_index: Index of the query to embed.
        """
        return self.get_embedding(int(query_index), 'question')
    
# Initialize the custom embeddings class
custom_embeddings = CustomListEmbeddings(ragas_data)

# Define the evaluation metrics
metrics = [faithfulness, context_utilization]

# Run the evaluation
evaluation_report = evaluate(dataset=ragas_data, metrics=metrics, embeddings=custom_embeddings)

# Print the evaluation results
print("RAGAS Evaluation Report:")
print(evaluation_report)