Repository: explodinggradients/ragas
Issue: evaluate not working
Root cause: External Dependency Breakage

**Describe the bug**
Following this exact steps to reproduce 
https://docs.ragas.io/en/stable/howtos/customisations/llms.html

Ragas version:
Python version:3.10

**Code to Reproduce**
https://docs.ragas.io/en/stable/howtos/customisations/llms.html
**Error trace**
---------------------------------------------------------------------------NotFoundError                             Traceback (most recent call last)
Cell In[7], line 1----> 1 result2 = evaluate(      2     fiqa_eval["baseline"].select(range(2)),  # showing only 5 for demonstration      3     metrics=[context_recall],      4 )      6 result2

File //auto-eval/ragas/evaluation.py:101, in evaluate(dataset, metrics, column_map)     99         binary_metrics.append(metric.name)    100     #print(f"evaluating with [{metric.name}]")--> 101     scores.append(metric.score(dataset).select_columns(metric.name))    103 # log the evaluation event    104 metrics_names = [m.name for m in metrics]

File auto-eval/ragas/metrics/base.py:76, in Metric.score(self, dataset, callbacks)     74 with trace_as_chain_group(f"ragas_{self.name}", callback_manager=cm) as group:     75     for batch in tqdm(self.get_batches(len(dataset))):---> 76         score = self._score_batch(dataset.select(batch), callbacks=group)     77         scores.extend(score)     79 return dataset.add_column(f"{self.name}", scores)

File /auto-eval/ragas/metrics/_context_recall.py:113, in ContextRecall._score_batch(self, dataset, callbacks, callback_group_name)    110     prompts.append(ChatPromptTemplate.from_messages([human_prompt]))    112 responses: list[list[str]] = []--> 113 results = self.llm.generate(    114     prompts,    115     n=1,    116     callbacks=batch_group,    117 )    118 responses = [[i.text for i in r] for r in results.generations]    119 scores = []

File/auto-eval/ragas/llms/langchain.py:209, in LangchainLLM.generate(self, prompts, n, temperature, callbacks)    206     self.llm.temperature = temperature    208 if self.llm_supports_completions(self.llm):--> 209     return self._generate_multiple_completions(prompts, n, callbacks)    210 else:  # call generate_completions n times to mimic multiple completions    211     list_llmresults = run_async_tasks(    212         [self.generate_completions(prompts, callbacks) for _ in range(n)]    213     )

File /auto-eval/ragas/llms/langchain.py:120, in LangchainLLM._generate_multiple_completions(self, prompts, n, callbacks)    118 else:  # if BaseChatModel    119     ps = [p.format_messages() for p in prompts]--> 120     result = self.llm.generate(ps, callbacks=callbacks)    121 self.llm.n = old_n    123 return result

File ~/.local/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:407, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, **kwargs)    405         if run_managers:    406             run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))--> 407         raise e    408 flattened_outputs = [    409     LLMResult(generations=[res.generations], llm_output=res.llm_output)    410     for res in results    411 ]    412 llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File ~/.local/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:397, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, **kwargs)    394 for i, m in enumerate(messages):    395     try:    396         results.append(--> 397             self._generate_with_cache(    398                 m,    399                 stop=stop,    400                 run_manager=run_managers[i] if run_managers else None,    401                 **kwargs,    402             )    403         )    404     except BaseException as e:    405         if run_managers:

File ~/.local/lib/python3.10/site-packages/langchain_core/language_models/chat_models.py:576, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)    572     raise ValueError(    573         "Asked to cache, but no cache found at `langchain.cache`."    574     )    575 if new_arg_supported:--> 576     return self._generate(    577         messages, stop=stop, run_manager=run_manager, **kwargs    578     )    579 else:    580     return self._generate(messages, stop=stop, **kwargs)

File ~/.local/lib/python3.10/site-packages/langchain_community/chat_models/openai.py:437, in ChatOpenAI._generate(self, messages, stop, run_manager, stream, **kwargs)    431 message_dicts, params = self._create_message_dicts(messages, stop)    432 params = {    433     **params,    434     **({"stream": stream} if stream is not None else {}),    435     **kwargs,    436 }--> 437 response = self.completion_with_retry(    438     messages=message_dicts, run_manager=run_manager, **params    439 )    440 return self._create_chat_result(response)

File ~/.local/lib/python3.10/site-packages/langchain_community/chat_models/openai.py:354, in ChatOpenAI.completion_with_retry(self, run_manager, **kwargs)    352 """Use tenacity to retry the completion call."""    353 if is_openai_v1():--> 354     return self.client.create(**kwargs)    356 retry_decorator = _create_retry_decorator(self, run_manager=run_manager)    358 @retry_decorator    359 def _completion_with_retry(**kwargs: Any) -> Any:

File ~/.local/lib/python3.10/site-packages/openai/_utils/_utils.py:271, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)    269             msg = f"Missing required argument: {quote(missing[0])}"    270     raise TypeError(msg)--> 271 return func(*args, **kwargs)
File ~/.local/lib/python3.10/site-packages/openai/resources/chat/completions.py:643, in Completions.create(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, presence_penalty, response_format, seed, stop, stream, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)    594 @required_args(["messages", "model"], ["messages", "model", "stream"])    595 def create(    596     self,   (...)    641     timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,    642 ) -> ChatCompletion | Stream[ChatCompletionChunk]:--> 643     return self._post(    644         "/chat/completions",    645         body=maybe_transform(    646             {    647                 "messages": messages,    648                 "model": model,    649                 "frequency_penalty": frequency_penalty,    650                 "function_call": function_call,    651                 "functions": functions,    652                 "logit_bias": logit_bias,    653                 "logprobs": logprobs,    654                 "max_tokens": max_tokens,    655                 "n": n,    656                 "presence_penalty": presence_penalty,    657                 "response_format": response_format,    658                 "seed": seed,    659                 "stop": stop,    660                 "stream": stream,    661                 "temperature": temperature,    662                 "tool_choice": tool_choice,    663                 "tools": tools,    664                 "top_logprobs": top_logprobs,    665                 "top_p": top_p,    666                 "user": user,    667             },    668             completion_create_params.CompletionCreateParams,    669         ),    670         options=make_request_options(    671             extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout    672         ),    673         cast_to=ChatCompletion,    674         stream=stream or False,    675         stream_cls=Stream[ChatCompletionChunk],    676     )
File ~/.local/lib/python3.10/site-packages/openai/_base_client.py:1112, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls)   1098 def post(   1099     self,   1100     path: str,   (...)   1107     stream_cls: type[_StreamT] | None = None,   1108 ) -> ResponseT | _StreamT:   1109     opts = FinalRequestOptions.construct(   1110         method="post", url=path, json_data=body, files=to_httpx_files(files), **options   1111     )-> 1112     return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))

File ~/.local/lib/python3.10/site-packages/openai/_base_client.py:859, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls)    850 def request(    851     self,    852     cast_to: Type[ResponseT],   (...)    857     stream_cls: type[_StreamT] | None = None,    858 ) -> ResponseT | _StreamT:--> 859     return self._request(    860         cast_to=cast_to,    861         options=options,    862         stream=stream,    863         stream_cls=stream_cls,    864         remaining_retries=remaining_retries,    865     )
File ~/.local/lib/python3.10/site-packages/openai/_base_client.py:949, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls)    946         err.response.read()    948     log.debug("Re-raising status error")--> 949     raise self._make_status_error_from_response(err.response) from None
    951 return self._process_response(
    952     cast_to=cast_to,
    953     options=options,
   (...)
    956     stream_cls=stream_cls,
    957 )

NotFoundError: Error code: 404 - {'detail': 'Not Found'}
**Expected behavior**
It should return the score of metrics


**Additional context**
Add any other context about the problem here.
installation done using : pip install ragas


The evaluate function also errors out when installing ragas using 
`pip install git+https://github.com/explodinggradients/ragas.git`

This is the error I got after following the [get_started guide](https://docs.ragas.io/en/latest/getstarted/evaluation.html).

Traceback (most recent call last):
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/evaluation.py", line 176, in evaluate
    raise e
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/evaluation.py", line 159, in evaluate
    results = executor.results()
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/executor.py", line 118, in results
    raise e
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/executor.py", line 114, in results
    r = future.result()
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/concurrent/futures/_base.py", line 451, in result
    return self.__get_result()
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/executor.py", line 36, in wrapped_callable
    return counter, callable(*args, **kwargs)
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/metrics/base.py", line 75, in score
    raise e
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/metrics/base.py", line 71, in score
    score = self._score(row=row, callbacks=group_cm)
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/metrics/_answer_relevance.py", line 119, in _score
    result = self.llm.generate_text(
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/llms/base.py", line 102, in generate_text
    return self.langchain_llm.generate_prompt(
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/base.py", line 459, in generate_prompt
    return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/base.py", line 349, in generate
    raise e
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/base.py", line 339, in generate
    self._generate_with_cache(
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/base.py", line 492, in _generate_with_cache
    return self._generate(
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/openai.py", line 415, in _generate
    message_dicts, params = self._create_message_dicts(messages, stop)
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/openai.py", line 430, in _create_message_dicts
    message_dicts = [convert_message_to_dict(m) for m in messages]
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/chat_models/openai.py", line 430, in <listcomp>
    message_dicts = [convert_message_to_dict(m) for m in messages]
  File "/home/mgmanzanillo3/.local/lib/python3.10/site-packages/langchain/adapters/openai.py", line 115, in convert_message_to_dict
    raise TypeError(f"Got unknown type {message}")
TypeError: Got unknown type content='Generate a question for the given answer and Identify if answer is noncommittal\n\nanswer: "Albert Einstein was born in Germany."\ncontext: "Albert Einstein was a German-born theoretical physicist who is widely held to be one of the greatest and most influential scientists of all time"\noutput: "{\\"question\\":\\"Where was Albert Einstein born?\\",\\"noncommittal\\":false}"\n\nanswer: "It can change its skin color based on the temperature of its environment."\ncontext: "A recent scientific study has discovered a new species of frog in the Amazon rainforest that has the unique ability to change its skin color based on the temperature of its environment."\noutput: "{\\"question\\":\\"What unique ability does the newly discovered species of frog have?\\",\\"noncommittal\\":false}"\n\nanswer: "Everest"\ncontext: "The tallest mountain on Earth, measured from sea level, is a renowned peak located in the Himalayas."\noutput: "{\\"question\\":\\"What is the tallest mountain on Earth?\\",\\"noncommittal\\":false}"\n\nanswer: "I don\'t know about the  groundbreaking feature of the smartphone invented in 2023 as am unware of information beyond 2022. "\ncontext: "In 2023, a groundbreaking invention was announced: a smartphone with a battery life of one month, revolutionizing the way people use mobile technology."\noutput: "{\\"question\\":\\"What was the groundbreaking feature of the smartphone invented in 2023?\\", \\"noncommittal\\":true}"\n\nanswer: \nThe best way to deposit a cheque issued to an associate in your business into your business account is to open a business account with the bank. You will need a state-issued "dba" certificate from the county clerk\'s office as well as an Employer ID Number (EIN) issued by the IRS. Once you have opened the business account, you can have the associate sign the back of the cheque and deposit it into the business account.\ncontext: Just have the associate sign the back and then deposit it.  It\'s called a third party cheque and is perfectly legal.  I wouldn\'t be surprised if it has a longer hold period and, as always, you don\'t get the money if the cheque doesn\'t clear. Now, you may have problems if it\'s a large amount or you\'re not very well known at the bank.  In that case you can have the associate go to the bank and endorse it in front of the teller with some ID.  You don\'t even technically have to be there.  Anybody can deposit money to your account if they have the account number. He could also just deposit it in his account and write a cheque to the business."I have checked with Bank of America, and they say the ONLY way to cash (or deposit, or otherwise get access to the funds represented by a check made out to my business) is to open a business account. They tell me this is a Federal regulation, and every bank will say the same thing.  To do this, I need a state-issued ""dba"" certificate (from the county clerk\'s office) as well as an Employer ID Number (EIN) issued by the IRS. AND their CHEAPEST business banking account costs $15 / month. I think I can go to the bank that the check is drawn upon, and they will cash it, assuming I have documentation showing that I am the sole proprietor. But I\'m not sure.... What a racket!!"When a business asks me to make out a cheque to a person rather than the business name, I take that as a red flag. Frankly it usually means that the person doesn\'t want the money going through their business account for some reason - probably tax evasion. I\'m not saying you are doing that, but it is a frequent issue. If the company makes the cheque out to a person they may run the risk of being party to fraud. Worse still they only have your word for it that you actually own the company, and aren\'t ripping off your employer by pocketing their payment. Even worse, when the company is audited and finds that cheque, the person who wrote it will have to justify and document why they made it out to you or risk being charged with embezzlement. It\'s very much in their interests to make the cheque out to the company they did business with. Given that, you should really have an account in the name of your business. It\'s going to make your life much simpler in the long run.\noutput: \n'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/storage/local/mgmanzanillo3/projects/netopia/bmpoc/ragas/gpt4Test.py", line 11, in <module>
    result = evaluate(
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/evaluation.py", line 178, in evaluate
    result = Result(
  File "<string>", line 6, in __init__
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/ragas/evaluation.py", line 207, in __post_init__
    for cn in self.scores[0].keys():
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/datasets/arrow_dataset.py", line 2800, in __getitem__
    return self._getitem(key)
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/datasets/arrow_dataset.py", line 2784, in _getitem
    pa_subtable = query_table(self._data, key, indices=self._indices if self._indices is not None else None)
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/datasets/formatting/formatting.py", line 583, in query_table
    _check_valid_index_key(key, size)
  File "/home/mgmanzanillo3/miniconda3/envs/jbmpocv1/lib/python3.10/site-packages/datasets/formatting/formatting.py", line 526, in _check_valid_index_key
    raise IndexError(f"Invalid key: {key} is out of bounds for size {size}")
IndexError: Invalid key: 0 is out of bounds for size 0

May I ask how to fix it?

from ragas.llms.langchain import LangchainLLM

ai = ChatOpenAI(model_name=model_name)
llm = LangchainLLM(llm=ai)