Repository: explodinggradients/ragas
Issue: AttributeError when using FactualCorrectness metric example from documentation
Root cause: Documentation Deficiency

[ ] I have checked the [documentation](https://docs.ragas.io/) and related resources and couldn't resolve my bug.

**Describe the bug**
I'm trying to use the example for the FactualCorrectness metric from the official RAGAS documentation, but I encountered an error : AttributeError: 'str' object has no attribute 'content'

Ragas version: 0.2.15
Python version: 3.10.16

**Code to Reproduce**
```
from ragas.dataset_schema import SingleTurnSample
from ragas.metrics._factual_correctness import FactualCorrectness


sample = SingleTurnSample(
    response="The Eiffel Tower is located in Paris.",
    reference="The Eiffel Tower is located in Paris. I has a height of 1000ft."
)

scorer = FactualCorrectness(llm = evaluator_llm)
await scorer.single_turn_ascore(sample)
```

**Error trace**
```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
*/agent-evaluation/metrics.ipynb Cell 2 line 1
      5 sample = SingleTurnSample(
      6     response="The Eiffel Tower is located in Paris.",
      7     reference="The Eiffel Tower is located in Paris. I has a height of 1000ft."
      8 )
     10 scorer = FactualCorrectness(llm = evaluator_llm)
---> 11 await scorer.single_turn_ascore(sample)

File */site-packages/ragas/metrics/base.py:541, in SingleTurnMetric.single_turn_ascore(self, sample, callbacks, timeout)
    539     if not group_cm.ended:
    540         rm.on_chain_error(e)
--> 541     raise e
    542 else:
    543     if not group_cm.ended:

File */site-packages/ragas/metrics/base.py:534, in SingleTurnMetric.single_turn_ascore(self, sample, callbacks, timeout)
    527 rm, group_cm = new_group(
    528     self.name,
    529     inputs=sample.to_dict(),
    530     callbacks=callbacks,
    531     metadata={"type": ChainType.METRIC},
    532 )
    533 try:
--> 534     score = await asyncio.wait_for(
    535         self._single_turn_ascore(sample=sample, callbacks=group_cm),
    536         timeout=timeout,
    537     )
    538 except Exception as e:
    539     if not group_cm.ended:

File /mnt/workspace/conda_envs/agent_evaluation/lib/python3.10/asyncio/tasks.py:408, in wait_for(fut, timeout)
    405 loop = events.get_running_loop()
    407 if timeout is None:
--> 408     return await fut
    410 if timeout <= 0:
    411     fut = ensure_future(fut, loop=loop)

File */site-packages/ragas/metrics/_factual_correctness.py:261, in FactualCorrectness._single_turn_ascore(self, sample, callbacks)
    258 assert reference is not None, "Reference is not set"
    259 assert response is not None, "Response is not set"
--> 261 response_claims = await self.decompose_claims(response, callbacks)
    262 reference_response = await self.verify_claims(
    263     premise=reference, hypothesis_list=response_claims, callbacks=callbacks
    264 )
    266 if self.mode != "precision":

File */site-packages/ragas/metrics/_factual_correctness.py:231, in FactualCorrectness.decompose_claims(self, response, callbacks)
    228 assert self.llm is not None, "LLM must be set"
    230 prompt_input = ClaimDecompositionInput(response=response)
--> 231 result = await self.claim_decomposition_prompt.generate(
    232     data=prompt_input, llm=self.llm, callbacks=callbacks
    233 )
    234 return result.claims

File */site-packages/ragas/prompt/pydantic_prompt.py:129, in PydanticPrompt.generate(self, llm, data, temperature, stop, callbacks, retries_left)
    126 callbacks = callbacks or []
    128 # this is just a special case of generate_multiple
--> 129 output_single = await self.generate_multiple(
    130     llm=llm,
    131     data=data,
    132     n=1,
    133     temperature=temperature,
    134     stop=stop,
    135     callbacks=callbacks,
    136     retries_left=retries_left,
    137 )
    138 return output_single[0]

File */site-packages/ragas/prompt/pydantic_prompt.py:191, in PydanticPrompt.generate_multiple(self, llm, data, n, temperature, stop, callbacks, retries_left)
    189 prompt_value = PromptValue(text=self.to_string(processed_data))
    190 # prompt_value = prompt_value.to_messages()
--> 191 resp = await llm.generate(
    192     prompt_value,
    193     n=n,
    194     temperature=temperature,
    195     stop=stop,
    196     callbacks=prompt_cb,
    197 )
    199 output_models = []
    200 parser = RagasOutputParser(pydantic_object=self.output_model)

File */site-packages/langchain_core/language_models/chat_models.py:757, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    743 inheritable_metadata = {
    744     **(metadata or {}),
    745     **self._get_ls_params(stop=stop, **kwargs),
    746 }
    748 callback_manager = CallbackManager.configure(
    749     callbacks,
    750     self.callbacks,
   (...)
    755     self.metadata,
    756 )
--> 757 messages_to_trace = [
    758     _format_for_tracing(message_list) for message_list in messages
    759 ]
    760 run_managers = callback_manager.on_chat_model_start(
    761     self._serialized,
    762     messages_to_trace,
   (...)
    767     batch_size=len(messages),
    768 )
    769 results = []

File */site-packages/langchain_core/language_models/chat_models.py:758, in <listcomp>(.0)
    743 inheritable_metadata = {
    744     **(metadata or {}),
    745     **self._get_ls_params(stop=stop, **kwargs),
    746 }
    748 callback_manager = CallbackManager.configure(
    749     callbacks,
    750     self.callbacks,
   (...)
    755     self.metadata,
    756 )
    757 messages_to_trace = [
--> 758     _format_for_tracing(message_list) for message_list in messages
    759 ]
    760 run_managers = callback_manager.on_chat_model_start(
    761     self._serialized,
    762     messages_to_trace,
   (...)
    767     batch_size=len(messages),
    768 )
    769 results = []

File */site-packages/langchain_core/language_models/chat_models.py:126, in _format_for_tracing(messages)
    124 for message in messages:
    125     message_to_trace = message
--> 126     if isinstance(message.content, list):
    127         for idx, block in enumerate(message.content):
    128             if (
    129                 isinstance(block, dict)
    130                 and block.get("type") == "image"
    131                 and is_data_content_block(block)
    132                 and block.get("source_type") != "id"
    133             ):

AttributeError: 'str' object has no attribute 'content'
```

Thanks in advance for any assistance you can provide


I know this might be caused by version incompatibility between ragas and langchain, langchain-core, etc. Could you please provide a compatible version dependency table?

Sorry, I found the issue — I missed the line evaluator_llm = LangchainLLMWrapper(llm_model). The problem is resolved now.

Thanks @glimmera , you saved my day