Repository: confident-ai/deepeval
Issue: Metric initialization not working with custom LLM (still expects OpenAI API keys)
Root cause: Validation Gap

**❗BEFORE YOU BEGIN❗**
Are you on discord? 🤗 We'd love to have you asking questions on discord instead: https://discord.com/invite/a3K9c8GRGt

**Describe the bug**

After following the [custom LLM setup documentation](https://docs.confident-ai.com/docs/metrics-introduction#using-a-custom-llm), when trying to initialize a metric, DeepEval seems to keep expecting an OpenAI API key which throws an exception.


**To Reproduce**
Steps to reproduce the behavior:

```
import pandas as pd
import torch

from deepeval.metrics.ragas import (
    RAGASContextualPrecisionMetric,
    RAGASFaithfulnessMetric,
    RAGASContextualRecallMetric,
    RAGASAnswerRelevancyMetric,
)

from deepeval.metrics import BiasMetric
from deepeval import evaluate
from deepeval.test_case import LLMTestCase
from transformers import AutoModelForCausalLM, AutoTokenizer
from deepeval.models.base_model import DeepEvalBaseLLM
from transformers import BitsAndBytesConfig

class Mixtral7Bby8(DeepEvalBaseLLM):
    def __init__(
            self,
            model,
            tokenizer
    ):
        self.model = model
        self.tokenizer = tokenizer

    def load_model(self):
        return self.model

    def generate(self, prompt: str) -> str:
        model = self.load_model()

        # device = "cuda" # the device to load the model onto

        model_inputs = self.tokenizer([prompt], return_tensors="pt").to("cuda")
        #model.to(device)

        generated_ids = model.generate(
            **model_inputs,
            max_new_tokens=156,
            pad_token_id=tokenizer.eos_token_id,
        )
        return self.tokenizer.decode(
            generated_ids[0],
            skip_special_tokens=True)

    async def a_generate(self, prompt: str) -> str:
        return self.generate(prompt)

    def get_model_name(self):
        return "Mixtral 7Bx8"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)


model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    torch_dtype=torch.float16,
    low_cpu_mem_usage=True,
)

mixtral_7bx8 = Mixtral7Bby8(
    model=model,
    tokenizer=tokenizer)

contextual_precision = RAGASContextualPrecisionMetric(
    threshold=0.5,
    model=mixtral_7bx8,
)

```

STACK TRACE

```

ValidationError: 2 validation errors for ChatOpenAI
model
  none is not an allowed value (type=type_error.none.not_allowed)
__root__
  Did not find openai_api_key, please add an environment variable `OPENAI_API_KEY` which contains it, or pass `openai_api_key` as a named parameter. (type=value_error)


--------------------------------------------------------------------------
ValidationError                           Traceback (most recent call last)
Cell In[6], line 1
----> 1 contextual_precision = RAGASContextualPrecisionMetric(
      2     threshold=0.5,
      3     model=mixtral_7bx8,
      4 )


File ~/miniconda3/envs/llm-env4/lib/python3.10/site-packages/deepeval/metrics/ragas.py:27, in RAGASContextualPrecisionMetric.__init__(self, threshold, model)
     21 def __init__(
     22     self,
     23     threshold: float = 0.3,
     24     model: Optional[Union[str, BaseChatModel]] = "gpt-3.5-turbo",
     25 ):
     26     self.threshold = threshold
---> 27     self.model = GPTModel(model=model)
     28     self.evaluation_model = self.model.get_model_name()

File ~/miniconda3/envs/llm-env4/lib/python3.10/site-packages/deepeval/models/gpt_model.py:42, in GPTModel.__init__(self, model, *args, **kwargs)
     39 elif model is None:
     40     model_name = default_gpt_model
---> 42 super().__init__(model_name, *args, **kwargs)

File ~/miniconda3/envs/llm-env4/lib/python3.10/site-packages/deepeval/models/base_model.py:35, in DeepEvalBaseLLM.__init__(self, model_name, *args, **kwargs)
     33 def __init__(self, model_name: Optional[str] = None, *args, **kwargs):
     34     self.model_name = model_name
---> 35     self.model = self.load_model(*args, **kwargs)

File ~/miniconda3/envs/llm-env4/lib/python3.10/site-packages/deepeval/models/gpt_model.py:75, in GPTModel.load_model(self)
     65         model_version = ""
     67     return AzureChatOpenAI(
     68         openai_api_version=openai_api_version,
     69         azure_deployment=azure_deployment,
   (...)
     72         model_version=model_version,
     73     )
---> 75 return ChatOpenAI(model_name=self.model_name)

File ~/miniconda3/envs/llm-env4/lib/python3.10/site-packages/langchain_core/load/serializable.py:120, in Serializable.__init__(self, **kwargs)
    119 def __init__(self, **kwargs: Any) -> None:
--> 120     super().__init__(**kwargs)
    121     self._lc_kwargs = kwargs

File ~/miniconda3/envs/llm-env4/lib/python3.10/site-packages/pydantic/v1/main.py:341, in BaseModel.__init__(__pydantic_self__, **data)
    339 values, fields_set, validation_error = validate_model(__pydantic_self__.__class__, data)
    340 if validation_error:
--> 341     raise validation_error
    342 try:
    343     object_setattr(__pydantic_self__, '__dict__', values)

ValidationError: 2 validation errors for ChatOpenAI
model
  none is not an allowed value (type=type_error.none.not_allowed)
__root__
  Did not find openai_api_key, please add an environment variable `OPENAI_API_KEY` which contains it, or pass `openai_api_key` as a named parameter. (type=value_error)

```


**Expected behavior**
I expected the metric object to use the custom LLM without asking for any OpenAI API tokens.  


**Screenshots**
Code and stack trace already provided


**Additional context**
Ubuntu 22.04
Python 3.10
Transformers 4.39.1
DeepEval 0.21.0

Same problem (I guess) here.

My code:

```
`class AWSBedrock(DeepEvalBaseLLM):
    def __init__(
        self,
        model
    ):
        self.model = model

    def load_model(self):
        return self.model

    def generate(self, prompt: str) -> str:
        chat_model = self.load_model()
        return chat_model.invoke(prompt).content

    async def a_generate(self, prompt: str) -> str:
        chat_model = self.load_model()
        res = await chat_model.ainvoke(prompt)
        return res.content

    def get_model_name(self):
        return "Custom Anthropic Bedrock Model"


bedrock = BedrockChat(
    model_id=BEDROCK_MODEL_ID, 
    region_name="us-east-1",
    #context_size=200000,
    #max_tokens=2048,
    #temperature=0,
    #top_p=0.1,
    )

aws_bedrock = AWSBedrock(model=bedrock)

# Answer Relevancy Metric

answer_relevancy_metric = AnswerRelevancyMetric(model=aws_bedrock)
answer_relevancy_metric.measure(test_case)

print(answer_relevancy_metric.score)
print(answer_relevancy_metric.reason)

# Model Evaluation

evaluator = DeepEvalAnswerRelevancyEvaluator(model=aws_bedrock)
evaluation_result = evaluator.evaluate_response(
    query=user_input,
    response=response_object
)

print(evaluation_result)

# Ragas Metric

ragas_metric = RagasMetric(model=aws_bedrock)
ragas_metric.measure(test_case)
print(ragas_metric.score)

#evaluate([test_case], [ragas_metric])
`
```

Everything is working nice but RagasMetric. Keep getting:
```
`pydantic.v1.error_wrappers.ValidationError: 1 validation error for ChatOpenAI
model
  none is not an allowed value (type=type_error.none.not_allowed)
```
`

per DeepEval maintainer's guidance. RAGAS evaluators only work with LAngChain chat models and OpenAI are probably the only ones capable of returning valid JSON responses. So closing the issue as I was misusing the tool.