Repository: explodinggradients/ragas
Issue: Adding vLLM models support for evaluation
Root cause: Unimplemented Feature Gap

**Describe the Feature**
LlamaIndex and Langchain wrapped models are available for evaluation, Langchain contains a wrapper for vLLM but it is not the most up-to-date and has some issues. So it would be interesting to add a native support for vLLM.

**Why is the feature important for you?**
I couldn't use vLLM with Langchain

**Additional context**
I already wrote a code that works.

```python
import os
os.environ["OUTLINES_CACHE_DIR"] = "vllm_cache"
os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'  # fork method doesn't work with when cuda is already initialized

import typing as t

from ragas.metrics import LLMContextPrecisionWithReference, LLMContextRecall
from ragas import evaluate, EvaluationDataset
from vllm import AsyncLLMEngine, LLM, SamplingParams
from vllm.engine.arg_utils import AsyncEngineArgs


import uuid

from langchain_core.callbacks import Callbacks
from langchain_core.outputs import LLMResult, Generation
from langchain_core.prompt_values import PromptValue
from ragas.cache import CacheInterface
from ragas.llms import BaseRagasLLM
from ragas.run_config import RunConfig


class vLLMWrapper(BaseRagasLLM):
    """
    A wrapper class that adapts vLLM's inference engine to the Ragas-compatible BaseRagasLLM interface.

    This class enables using vLLM for scoring and evaluation tasks within the Ragas framework by implementing
    the `generate_text` and `agenerate_text` method that produces LangChain-compatible `LLMResult` objects.
    Source: https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/llms/base.py#L123

    Attributes:
        llm: The vLLM model instance, typically created via `vllm.LLM(...)`.
        sampling_params: A `SamplingParams` object defining temperature, top_p, etc.
        run_config: Optional configuration for controlling how evaluations are executed.
        cache: Optional cache for storing/reusing model outputs.

    """

    def __init__(
        self,
        vllm_model,
        sampling_params,
        run_config: t.Optional[RunConfig] = None,
        cache: t.Optional[CacheInterface] = None,
    ):
        super().__init__(cache=cache)
        self.llm = vllm_model
        self.sampling_params = sampling_params
        
        if run_config is None:  # legacy code 
            run_config = RunConfig()
        self.set_run_config(run_config)

    def is_finished(self, response: LLMResult) -> bool:
        """
        Verify that generation finished correctly by looking at finish_reason.
        `response` contains the n outputs of a single input, thus:
            len(response.generations) == 1
            len(response.generations[0]) == n
        """
        is_finished_list = []
        for single_generation in response.generations[0]:
            # generation_info is provided with `finish_reason`
            finish_reason = single_generation.generation_info.get("finish_reason")
            is_finished_list.append(finish_reason == 'stop')

        # if all the n outputs finished correctly, return True
        return all(is_finished_list)

    def generate_text(
        self,
        prompt: PromptValue,
        n: int = 1,
        temperature: t.Optional[float] = None,
        stop: t.Optional[t.List[str]] = None,
        callbacks: Callbacks = None,
    ) -> LLMResult:
        """
        Generates a LangChain-compatible LLMResult from a PromptValue using vLLM.

        This method is designed to be compatible with the BaseRagasLLM interface. It uses the
        preconfigured vLLM engine and sampling parameters to produce completions for a given prompt.

        Args:
            prompt (PromptValue): The input prompt wrapped in a LangChain PromptValue.
            n (int): The number of outputs for the prompt.

        Returns:
            LLMResult: A LangChain LLMResult containing one Generation per prompt.
        """
        # expected arguments from BaseRagasLLM that is kept to have a compatible API
        temperature = None
        stop = None
        callbacks = None

        prompt = prompt.to_string()  # vLLM requires a text as an input
        sampling_params.n = n        # generate n outputs per input
        sampling_params.best_of = n

        # vLLM engine will always produce a list[vllm.outputs.RequestOutput]
        # since we only have 1 prompt, the list has 1 entry
        vllm_result = self.llm.generate(prompt, self.sampling_params)[0]
        
        # LangChain's LLMResult expects a list of lists:
        # - The outer list corresponds to each input prompt
        # - The inner list contains one or more Generations per prompt (e.g. multiple outputs for a single input)
        # - We register the reason why the generation ended: stop, length, abort
        # source: https://docs.vllm.ai/en/stable/api/vllm/v1/engine/index.html?h=finish_reason#vllm.v1.engine.FINISH_REASON_STRINGS
        generations = [
            [Generation(text=output.text.strip(), generation_info={'finish_reason': output.finish_reason}) for output in vllm_result.outputs]
        ]
        ragas_expected_result = LLMResult(generations=generations)

        return ragas_expected_result

    async def agenerate_text(
        self,
        prompt: PromptValue,
        n: int = 1,
        temperature: t.Optional[float] = None,
        stop: t.Optional[t.List[str]] = None,
        callbacks: Callbacks = None,
    ) -> LLMResult:
        
        # expected arguments from BaseRagasLLM that is kept to have a compatible API
        temperature = None
        stop = None
        callbacks = None

        prompt = prompt.to_string()     # vLLM requires a text as an input
        sampling_params.n = n           # generate n outputs per input
        sampling_params.best_of = n
        request_id = str(uuid.uuid4())  # id used for tracking purpose
        # non-blocking calls, create a request to vLLM engine
        results_generator = self.llm.generate(prompt, self.sampling_params, request_id=request_id)
        
        # waiting for results from AsyncLLMEngine, while waiting, other coroutines will be working
        vllm_result= None
        async for request_output in results_generator:
            vllm_result = request_output
        
        # LangChain's LLMResult expects a list of lists:
        # - The outer list corresponds to each input prompt
        # - The inner list contains one or more Generations per prompt (e.g. multiple outputs for a single input)
        # - We register the reason why the generation ended: stop, length, abort
        # source: https://docs.vllm.ai/en/stable/api/vllm/v1/engine/index.html?h=finish_reason#vllm.v1.engine.FINISH_REASON_STRINGS
        generations = [
            [Generation(text=output.text.strip(), generation_info={'finish_reason': output.finish_reason}) for output in vllm_result.outputs]
        ]
        ragas_expected_result = LLMResult(generations=generations)

        return ragas_expected_result

    def set_run_config(self, run_config: RunConfig):
        self.run_config = run_config

    def __repr__(self) -> str:
        return f"{self.__class__.__name__}(llm={self.llm.__class__.__name__}(...))"

if __name__ == "__main__":
    
    judge_llm = AsyncLLMEngine.from_engine_args(
        AsyncEngineArgs(
            model='meta-llama/Llama-3.1-8B-Instruct',
            task='generate',              # generation task
            tokenizer_mode="auto",        # use fast implementation in Rust if available
            skip_tokenizer_init=False,    # use tokenizer and detokenizer
            trust_remote_code=False,
            dtype='bfloat16',
            quantization=None,            # "awq", "gptq", "fp8"
            compilation_config=3,
            enforce_eager=False,          # eager mode disable CUDA graph
            max_seq_len_to_capture=8192,  # fall back to eager mode if seq_len > 8192
            tensor_parallel_size=4,
            gpu_memory_utilization=0.9,
            swap_space=4,                 # offloading of KV cache if necessary
            cpu_offload_gb=0,             # no model offloading on CPU
            max_model_len=32768,          # set max number of tokens per sequence (prefill + decode) to 2^15
            enable_chunked_prefill=True,  # split batch of sequences to avoid potential vRAM overflow
            max_num_batched_tokens=32768  # threshold on total input tokens (num_seq * seq_len) before chunking
        )
    )  # Load LLM with tensor_parallel_size=4 & bf16 quantization (expected 4 GPUs)
    
    
    metrics = [  # these metrics relies either on a LLM or on an embedding model (EMB)
        LLMContextPrecisionWithReference(),   # Do we have all relevant contexts ? (LLM)
        LLMContextRecall(),                   # Do we have only relevant contexts or is there a lot of noises ? (LLM)
    ]
    
    
    dataset_ragas = [
        {
            'user_input': "What is the effect of adding stearic acid to PAO4 lubricant on the viscous damping friction contribution in steel/steel contacts?",
            'retrieved_contexts': [
                "The viscous damping friction value ߤଵ corresponding to PAO4 and 150NS lubricated and loaded steel/steel contacts is considered as negligible at any temperature since its corresponding value is less than that of the apparatus.",
                "When additives are used in PAO4 lubricant, the damping behavior of the system is reduced.",
                "Three different additives are tested with PAO4: oleic acid, linoleic acid, and stearic acid. Results show that linoleic acid has the lowest µ0; however, stearic acid has the lowest ߤଵ."
            ],
            'response': "Adding stearic acid to PAO4 lubricant reduces the viscous damping friction contribution in steel/steel contacts.",
            'reference': "Stearic acid reduces viscous damping friction more than oleic acid and linoleic acid, as it results in the lowest ߤଵ value among tested additives."
        }
    ]
    
    dataset_ragas = EvaluationDataset.from_list(dataset_ragas)
    
    # Load Judge LLM with its sampling parameters
    sampling_params = SamplingParams(
        temperature=0.6,
        top_p=0.9,
        max_tokens=2024,
    )  # Base sampling params from: https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct/blob/main/generation_config.json
    
    results = evaluate(dataset=dataset_ragas,
                       metrics=metrics,
                       llm=vLLMWrapper(judge_llm, sampling_params),
                       raise_exceptions=False,
                       run_config=RunConfig(timeout=180, max_retries=5, max_wait=60, max_workers=30),
                       show_progress=True)
```