Repository: explodinggradients/ragas
Issue: ValidationError with OpenRouter models
Root cause: External Dependency Breakage

I am encountering a pydantic_core.ValidationError when running the context_recall and Faithfulness metrics using most of Open Router models (like google/gemini-2.5-flash, qwen/qwen3-next-80b-a3b-instruct). I am using the 0.4.0 Ragas version.

The issue occurs because the LLM returns the items inside the classifications list as JSON strings rather than JSON objects (dictionaries). The Pydantic model expects a list of objects, but receives a list of strings, causing the validation to fail even after retries by the instructor library.

The code I am using:

import os
from openai import AsyncOpenAI
from ragas.llms import llm_factory
from ragas.metrics.collections import (
    ContextPrecision,
    ContextRecall,
    Faithfulness,
    AnswerRelevancy,
)
from ragas.embeddings import embedding_factory

class EmbeddingProviderOpenRouter:
    def __init__(self):
        self.client = AsyncOpenAI(
            base_url="https://openrouter.ai/api/v1",
            api_key=os.getenv("OPEN_ROUTER"),
        )
        self.model = "qwen/qwen3-embedding-8b"

        # cria embedder
        self.embedder = embedding_factory(
            provider="openai",               
            model=self.model,             
            client=self.client,               
            interface="modern",               
        )

    async def aembed(self, texts: list[str]) -> list[list[float]]:
        return await self.embedder.aembed(texts)

class RagasEvaluator:
    def __init__(self, model_name="google/gemini-2.5-flash"):
        self.model_name = model_name

        # Cliente OpenRouter
        self.client = AsyncOpenAI(
            base_url="https://openrouter.ai/api/v1",
            api_key=os.getenv("OPEN_ROUTER"),
        )
        #self.client = AsyncOpenAI(
        #    api_key=os.getenv("OPENAI_API_KEY"),
        #)

        # LLM usado pelo Ragas
        self.llm = llm_factory(
            model_name,
            provider="openai",
            client=self.client,
        )


    #MÉTRICAS

    async def context_precision(self, user_query: str, reference: str, retrieved_contexts: list[str]):
        """
        Avalia a precisão do contexto recuperado em relação à referência fornecida.
        """
        scorer = ContextPrecision(llm=self.llm)
        result = await scorer.ascore(
            user_input=user_query,
            reference=reference,
            retrieved_contexts=retrieved_contexts
        )
        return result.value

    async def context_recall(self, user_query: str, reference: str, retrieved_contexts: list[str]):
        """
        Avalia se o contexto recuperado cobre as informações necessárias para responder à pergunta do usuário.
        """
        scorer = ContextRecall(llm=self.llm)
        result = await scorer.ascore(
            user_input=user_query,
            reference=reference,
            retrieved_contexts=retrieved_contexts
        )
        return result.value

    async def faithfulness(self, user_query: str, response: str, retrieved_contexts: list[str]):
        """
        Avalia se a resposta do modelo é fiel ao contexto recuperado.
        """
        scorer = Faithfulness(llm=self.llm)
        result = await scorer.ascore(
            user_input=user_query,
            response=response,
            retrieved_contexts=retrieved_contexts
        )
        return result.value 
      
    async def answer_relevance(self, user_query: str, response: str):
        """
        Avalia se a resposta do modelo é relevante para a pergunta do usuário.
        """
        embedding_provider = EmbeddingProviderOpenRouter()
        
        scorer = AnswerRelevancy(
            llm=self.llm,
            embeddings = embedding_provider.embedder,
        )

        result = await scorer.ascore(
            user_input=user_query,
            response=response,
        )

        return result.value

###The errors messages:

Faithfulness: 

pydantic_core._pydantic_core.ValidationError: 30 validation errors for NLIStatementOutput
statements.0
  Input should be an object [type=model_type, input_value='Os incentivos fiscais vi... da Região Amazônica.', input_type=str]
    For further information visit https://errors.pydantic.dev/2.12/v/model_type

ContextRecall:

pydantic_core._pydantic_core.ValidationError: 8 validation errors for ContextRecallOutput
classifications.0
  Input should be an object [type=model_type, input_value='{"statement": "Os incent...nto.", "attributed": 1}', input_type=str]
    For further information visit https://errors.pydantic.dev/2.12/v/model_type

### Note:

It works fine with OpenAI models via OpenRouter, such as the openai/gpt-4o-mini model.

 Calculando context precision...
2025-12-08 12:38:16,197 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
2025-12-08 12:38:19,142 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
2025-12-08 12:38:21,475 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
context precision: 0.9999999999666667
🔍 Calculando context recall...
2025-12-08 12:38:23,668 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
context recall: 0.625
🔍 Calculando answer relevance...
2025-12-08 12:38:40,036 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
2025-12-08 12:38:42,207 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
2025-12-08 12:38:43,807 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
2025-12-08 12:38:45,223 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/embeddings "HTTP/1.1 200 OK"
2025-12-08 12:38:47,390 - INFO - Retrying request to /embeddings in 0.400924 seconds
2025-12-08 12:38:48,221 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/embeddings "HTTP/1.1 200 OK"
answer relevance: 0.6904672098781338
🔍 Calculando faithfulness...
2025-12-08 12:38:51,657 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
2025-12-08 12:38:59,890 - INFO - HTTP Request: POST https://openrouter.ai/api/v1/chat/completions "HTTP/1.1 200 OK"
faithfulness: 1.0
INFO:     127.0.0.1:51577 - "POST /evaluate-metrics HTTP/1.1" 200 OK

Same issue here. I know it is a free tool, but I don't understand the decision to favour OpenAI models over others. If you develop primarily against some models, at least pick the top of the leaderboard, like Claude or Gemini, make sure it works against those first. 

Hi @fredstrey 

I reproduced this on `ragas==0.4.0` (the version you mentioned) and compared it with `ragas==0.4.2` (current on PyPI). On my side, the **ValidationError persists on 0.4.0** but **does not occur on 0.4.2**. 

Environment:
- Python 3.13
- OpenRouter via `openai.AsyncOpenAI(base_url="https://openrouter.ai/api/v1", api_key=...)` 

On `ragas==0.4.0`, I see the same failure mode (nested JSON objects returned as JSON-encoded strings in `classifications` / `statements`), e.g.:

```text
context_recall ERROR: InstructorRetryException(1 validation error for ContextRecallClassifications
classifications.0
  Input should be an object [type=model_type, input_value='{"statement": "The Eiffe...ext.", "attributed": 1}', input_type=str]
  For further information visit https://errors.pydantic.dev/2.12/v/model_type)

faithfulness ERROR: InstructorRetryException(2 validation errors for NLIStatementOutput
...)
```


On `ragas==0.4.2`, the same script runs without errors for me (returns numeric scores). 

Models tested:
- `google/gemini-2.5-flash`: fails on 0.4.0, works on 0.4.2 
- `qwen/qwen3-next-80b-a3b-instruct`: I couldn’t reproduce the crash in my minimal script (both versions ran) 
- `openai/gpt-4o-mini`: works (both versions) 

Minimal repro script:

```python
import os
import asyncio
from dotenv import load_dotenv

from openai import AsyncOpenAI

from ragas.dataset_schema import SingleTurnSample
from ragas.llms import llm_factory
from ragas.metrics import LLMContextRecall, Faithfulness


def build_samples() -> list[tuple[str, SingleTurnSample]]:
    sample_a = SingleTurnSample(
        user_input="Where is the Eiffel Tower located?",
        response="The Eiffel Tower is located in Paris.",
        reference="The Eiffel Tower is located in Paris.",
        retrieved_contexts=["Paris is the capital of France."],
    )

    sample_b = SingleTurnSample(
        user_input="Where is the Eiffel Tower located?",
        response="The Eiffel Tower is located on the Champ de Mars in Paris, France.",
        reference="The Eiffel Tower is located on the Champ de Mars in Paris, France.",
        retrieved_contexts=[
            "The Eiffel Tower is located on the Champ de Mars in Paris, France."
        ],
    )

    return [("A_weak_context", sample_a), ("B_strong_context", sample_b)]


async def run_for_model(model_name: str, client: AsyncOpenAI) -> None:
    llm = llm_factory(model_name, provider="openai", client=client)

    ctx_recall = LLMContextRecall(llm=llm)
    faithfulness = Faithfulness(llm=llm)

    print("\n==============================")
    print(f"Model: {model_name}")
    print("==============================")

    for label, sample in build_samples():
        print(f"\n--- Sample {label} ---")
        print("retrieved_contexts:", sample.retrieved_contexts)

        try:
            cr = await ctx_recall.single_turn_ascore(sample)
            print("context_recall:", float(cr))
        except Exception as e:
            print("context_recall ERROR:", repr(e))

        try:
            f = await faithfulness.single_turn_ascore(sample)
            print("faithfulness:", float(f))
        except Exception as e:
            print("faithfulness ERROR:", repr(e))


async def main():
    load_dotenv()

    api_key = os.getenv("OPENROUTER_API_KEY")
    if not api_key:
        raise RuntimeError("OPENROUTER_API_KEY missing. Put it in .env")

    models = [
        os.getenv("OPENROUTER_MODEL_1", "google/gemini-2.5-flash"),
        os.getenv("OPENROUTER_MODEL_2", "qwen/qwen3-next-80b-a3b-instruct"),
        os.getenv("OPENROUTER_MODEL_3", "openai/gpt-4o-mini"),
    ]

    client = AsyncOpenAI(
        base_url="https://openrouter.ai/api/v1",
        api_key=api_key,
    )

    for m in models:
        await run_for_model(m, client)


if __name__ == "__main__":
    asyncio.run(main())
```

Could you try upgrading to `ragas>=0.4.2` and see if it fixes the issue for your setup? If it still fails on 0.4.2+, I can try reproducing with a more complex sample.

Thanks for checking this out @cgaswin 🙌🏼