Repository: modelscope/evalscope
Issue: RAG 评测出问题
Root cause: Interface Contract Mismatch

报错信息如下：
(evalscope) root@instance-duek25v4:~/fs/RAG# python test3.py 
2025-07-22 22:52:58,013 - evalscope - INFO - 开始评测RAG系统...
2025-07-22 22:52:58,013 - evalscope - INFO - Args: Task config is provided with dictionary type.
2025-07-22 22:52:58,018 - evalscope - INFO - Dump task config to ./outputs/20250722_225258/configs/task_config_73e069.yaml
2025-07-22 22:52:58,021 - evalscope - INFO - {
    "model": "DummyCustomModel",
    "model_id": "dummy-model",
    "model_args": {},
    "model_task": "text_generation",
    "template_type": null,
    "chat_template": null,
    "datasets": [],
    "dataset_args": {},
    "dataset_dir": "/root/.cache/modelscope/hub/datasets",
    "dataset_hub": "modelscope",
    "generation_config": {},
    "eval_type": "custom",
    "eval_backend": "RAGEval",
    "eval_config": {
        "tool": "RAGAS",
        "eval": {
            "testset_file": "/root/fs/RAG/converted_output.json",
            "critic_llm": {
                "model_name_or_path": "/root/fs/models//Qwen2.5-7B-Instruct"
            },
            "embeddings": {
                "model_name_or_path": "AI-ModelScope/bge-large-zh"
            },
            "metrics": [
                "Faithfulness",
                "AnswerRelevancy",
                "ContextPrecision",
                "AnswerCorrectness"
            ],
            "language": "chinese"
        }
    },
    "stage": "all",
    "limit": null,
    "eval_batch_size": 1,
    "mem_cache": false,
    "use_cache": null,
    "work_dir": "./outputs/20250722_225258",
    "outputs": null,
    "ignore_errors": false,
    "debug": false,
    "dry_run": false,
    "seed": 42,
    "api_url": null,
    "api_key": "EMPTY",
    "timeout": null,
    "stream": false,
    "judge_strategy": "auto",
    "judge_worker_num": 1,
    "judge_model_args": {},
    "analysis_report": false
}
2025-07-22 22:52:58,129 - evalscope - INFO - Check `ragas` Installed
Traceback (most recent call last):
  File "/root/fs/RAG/test3.py", line 36, in <module>
    evaluate_rag()
  File "/root/fs/RAG/test3.py", line 32, in evaluate_rag
    run_task(task_cfg=eval_task_cfg)
  File "/root/data/evalscope/evalscope/run.py", line 31, in run_task
    return run_single_task(task_cfg, run_time)
  File "/root/data/evalscope/evalscope/run.py", line 42, in run_single_task
    result = run_non_native_backend(task_cfg, outputs)
  File "/root/data/evalscope/evalscope/run.py", line 92, in run_non_native_backend
    backend_manager.run()
  File "/root/data/evalscope/evalscope/backend/rag_eval/backend_manager.py", line 87, in run
    self.run_ragas(testset_args, eval_args)
  File "/root/data/evalscope/evalscope/backend/rag_eval/backend_manager.py", line 64, in run_ragas
    rag_eval(EvaluationArguments(**eval_args))
  File "/root/data/evalscope/evalscope/backend/rag_eval/ragas/task_template.py", line 26, in rag_eval
    llm = LLM.load(**args.critic_llm)
  File "/root/data/evalscope/evalscope/backend/rag_eval/utils/llm.py", line 24, in load
    return LocalLLM(**kw)
  File "/root/data/evalscope/evalscope/backend/rag_eval/utils/llm.py", line 38, in __init__
    super().__init__(**kw)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/load/serializable.py", line 130, in __init__
    super().__init__(*args, **kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/pydantic/main.py", line 253, in __init__
    validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
pydantic_core._pydantic_core.ValidationError: 3 validation errors for LocalLLM
model_name
  Field required [type=missing, input_value={'model_name_or_path': '/...s//Qwen2.5-7B-Instruct'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.11/v/missing
model
  Field required [type=missing, input_value={'model_name_or_path': '/...s//Qwen2.5-7B-Instruct'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.11/v/missing
generation_config
  Field required [type=missing, input_value={'model_name_or_path': '/...s//Qwen2.5-7B-Instruct'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.11/v/missing


我用的代码：
from evalscope.run import run_task
from evalscope.utils.logger import get_logger
from evalscope.backend.rag_eval.utils.llm import LocalLLM

logger = get_logger()

def evaluate_rag():
    eval_task_cfg = {
        "eval_backend": "RAGEval",
        "eval_config": {
            "tool": "RAGAS",
            "eval": {
                "testset_file": "/root/fs/RAG/converted_output.json",
                 "critic_llm": {
                    "model_name_or_path": "/root/fs/models//Qwen2.5-7B-Instruct",
            },
                "embeddings": {
                    "model_name_or_path": "AI-ModelScope/bge-large-zh",  
                },
                "metrics": [
                    "Faithfulness",
                    "AnswerRelevancy",
                    "ContextPrecision",
                    "AnswerCorrectness"
                ],
                "language": "chinese"
            }
        },
    }

    logger.info("开始评测RAG系统...")
    run_task(task_cfg=eval_task_cfg)
    logger.info("评测完成！")

if __name__ == "__main__":
    evaluate_rag()

evalscope版本2.0.0 ragas版本0.2.14

"model_name_or_path": "/root/fs/models//Qwen2.5-7B-Instruct", 是不是写错了，多了一个斜杠

改了，还是一样的报错
2025-07-23 10:28:25,938 - evalscope - INFO - 开始评测RAG系统...
2025-07-23 10:28:25,938 - evalscope - INFO - Args: Task config is provided with dictionary type.
2025-07-23 10:28:25,961 - evalscope - INFO - Dump task config to ./outputs/20250723_102825/configs/task_config_8951d4.yaml
2025-07-23 10:28:25,966 - evalscope - INFO - {
    "model": "DummyCustomModel",
    "model_id": "dummy-model",
    "model_args": {},
    "model_task": "text_generation",
    "template_type": null,
    "chat_template": null,
    "datasets": [],
    "dataset_args": {},
    "dataset_dir": "/root/.cache/modelscope/hub/datasets",
    "dataset_hub": "modelscope",
    "generation_config": {},
    "eval_type": "custom",
    "eval_backend": "RAGEval",
    "eval_config": {
        "tool": "RAGAS",
        "eval": {
            "testset_file": "/root/fs/RAG/converted_output.json",
            "critic_llm": {
                "model_name_or_path": "/root/fs/models/Qwen2.5-7B-Instruct"
            },
            "embeddings": {
                "model_name_or_path": "AI-ModelScope/bge-large-zh"
            },
            "metrics": [
                "Faithfulness",
                "AnswerRelevancy",
                "ContextPrecision",
                "AnswerCorrectness"
            ],
            "language": "chinese"
        }
    },
    "stage": "all",
    "limit": null,
    "eval_batch_size": 1,
    "mem_cache": false,
    "use_cache": null,
    "work_dir": "./outputs/20250723_102825",
    "outputs": null,
    "ignore_errors": false,
    "debug": false,
    "dry_run": false,
    "seed": 42,
    "api_url": null,
    "api_key": "EMPTY",
    "timeout": null,
    "stream": false,
    "judge_strategy": "auto",
    "judge_worker_num": 1,
    "judge_model_args": {},
    "analysis_report": false
}
2025-07-23 10:28:26,124 - evalscope - INFO - Check `ragas` Installed
Traceback (most recent call last):
  File "/root/fs/RAG/test3.py", line 36, in <module>
    evaluate_rag()
  File "/root/fs/RAG/test3.py", line 32, in evaluate_rag
    run_task(task_cfg=eval_task_cfg)
  File "/root/data/evalscope/evalscope/run.py", line 31, in run_task
    return run_single_task(task_cfg, run_time)
  File "/root/data/evalscope/evalscope/run.py", line 42, in run_single_task
    result = run_non_native_backend(task_cfg, outputs)
  File "/root/data/evalscope/evalscope/run.py", line 92, in run_non_native_backend
    backend_manager.run()
  File "/root/data/evalscope/evalscope/backend/rag_eval/backend_manager.py", line 87, in run
    self.run_ragas(testset_args, eval_args)
  File "/root/data/evalscope/evalscope/backend/rag_eval/backend_manager.py", line 64, in run_ragas
    rag_eval(EvaluationArguments(**eval_args))
  File "/root/data/evalscope/evalscope/backend/rag_eval/ragas/task_template.py", line 26, in rag_eval
    llm = LLM.load(**args.critic_llm)
  File "/root/data/evalscope/evalscope/backend/rag_eval/utils/llm.py", line 24, in load
    return LocalLLM(**kw)
  File "/root/data/evalscope/evalscope/backend/rag_eval/utils/llm.py", line 38, in __init__
    super().__init__(**kw)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/load/serializable.py", line 130, in __init__
    super().__init__(*args, **kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/pydantic/main.py", line 253, in __init__
    validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
pydantic_core._pydantic_core.ValidationError: 3 validation errors for LocalLLM
model_name
  Field required [type=missing, input_value={'model_name_or_path': '/...ls/Qwen2.5-7B-Instruct'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.11/v/missing
model
  Field required [type=missing, input_value={'model_name_or_path': '/...ls/Qwen2.5-7B-Instruct'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.11/v/missing
generation_config
  Field required [type=missing, input_value={'model_name_or_path': '/...ls/Qwen2.5-7B-Instruct'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.11/v/missing

拉一下main分支代码，再尝试运行

拉取了，重新运行报了如下错误：

Traceback (most recent call last):
  File "/root/fs/RAG/test3.py", line 36, in <module>
    evaluate_rag()
  File "/root/fs/RAG/test3.py", line 32, in evaluate_rag
    run_task(task_cfg=eval_task_cfg)
  File "/root/data/evalscope/evalscope/run.py", line 31, in run_task
    return run_single_task(task_cfg, run_time)
  File "/root/data/evalscope/evalscope/run.py", line 42, in run_single_task
    result = run_non_native_backend(task_cfg, outputs)
  File "/root/data/evalscope/evalscope/run.py", line 92, in run_non_native_backend
    backend_manager.run()
  File "/root/data/evalscope/evalscope/backend/rag_eval/backend_manager.py", line 88, in run
    self.run_ragas(testset_args, eval_args)
  File "/root/data/evalscope/evalscope/backend/rag_eval/backend_manager.py", line 65, in run_ragas
    rag_eval(EvaluationArguments(**eval_args))
  File "/root/data/evalscope/evalscope/backend/rag_eval/ragas/task_template.py", line 31, in rag_eval
    asyncio.run(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/nest_asyncio.py", line 30, in run
    return loop.run_until_complete(task)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/nest_asyncio.py", line 98, in run_until_complete
    return f.result()
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/futures.py", line 201, in result
    raise self._exception.with_traceback(self._exception_tb)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/tasks.py", line 234, in __step
    result = coro.throw(exc)
  File "/root/data/evalscope/evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py", line 55, in translate_prompts
    await asyncio.gather(*[translate_prompt(prompt, target_lang, llm, adapt_instruction) for prompt in prompts])
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/tasks.py", line 304, in __wakeup
    future.result()
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/tasks.py", line 232, in __step
    result = coro.send(None)
  File "/root/data/evalscope/evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py", line 36, in translate_prompt
    adapted_prompts = await prompt_user.adapt_prompts(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/ragas/prompt/mixin.py", line 78, in adapt_prompts
    adapted_prompt = await prompt.adapt(language, llm, adapt_instruction)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/ragas/prompt/pydantic_prompt.py", line 234, in adapt
    translated_strings = await translate_statements_prompt.generate(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/ragas/prompt/pydantic_prompt.py", line 129, in generate
    output_single = await self.generate_multiple(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/ragas/prompt/pydantic_prompt.py", line 190, in generate_multiple
    resp = await llm.generate(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/ragas/llms/base.py", line 109, in generate
    result = await agenerate_text_with_retry(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/asyncio/__init__.py", line 189, in async_wrapped
    return await copy(fn, *args, **kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/asyncio/__init__.py", line 111, in __call__
    do = await self.iter(retry_state=retry_state)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/asyncio/__init__.py", line 153, in iter
    result = await action(retry_state)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/_utils.py", line 99, in inner
    return call(*args, **kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/__init__.py", line 420, in exc_check
    raise retry_exc.reraise()
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/__init__.py", line 187, in reraise
    raise self.last_attempt.result()
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/concurrent/futures/_base.py", line 451, in result
    return self.__get_result()
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/tenacity/asyncio/__init__.py", line 114, in __call__
    result = await fn(*args, **kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/ragas/llms/base.py", line 260, in agenerate_text
    result = await self.langchain_llm.agenerate_prompt(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/language_models/llms.py", line 777, in agenerate_prompt
    return await self.agenerate(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/language_models/llms.py", line 1231, in agenerate
    return await self._agenerate_helper(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/language_models/llms.py", line 1045, in _agenerate_helper
    await self._agenerate(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/language_models/llms.py", line 1566, in _agenerate
    await self._acall(prompt, stop=stop, run_manager=run_manager, **kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/language_models/llms.py", line 1525, in _acall
    return await run_in_executor(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/runnables/config.py", line 616, in run_in_executor
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/futures.py", line 285, in __await__
    yield self  # This tells Task to wait for completion.
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/tasks.py", line 304, in __wakeup
    future.result()
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/asyncio/futures.py", line 201, in result
    raise self._exception.with_traceback(self._exception_tb)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/miniconda3/envs/evalscope/lib/python3.10/site-packages/langchain_core/runnables/config.py", line 607, in wrapper
    return func(*args, **kwargs)
  File "/root/data/evalscope/evalscope/backend/rag_eval/utils/llm.py", line 60, in _call
    response, _ = self.model.predict([{'data': [prompt]}], infer_cfg=infer_cfg)
ValueError: not enough values to unpack (expected 2, got 1)