Tony20100/langchain_full huggingface dataset:
**kwargs (Any): Additional keyword arguments.
        Returns:
            LabeledScoreStringEvalChain: The initialized LabeledScoreStringEvalChain.
        Raises:
            ValueError: If the input variables are not as expected.
        """
        expected_input_vars = {
            "prediction",
            "input",
            "reference",
            "criteria",
        }
        prompt_ = prompt or SCORING_TEMPLATE_WITH_REFERENCE
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        criteria_ = resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items()).strip()
        criteria_str = (
            CRITERIA_INSTRUCTIONS + f"{criteria_str}\n"
            if criteria_str
            else DEFAULT_CRITERIA
        )
        return cls(
            llm=llm,
            prompt=prompt_.partial(criteria=criteria_str),
            normalize_by=normalize_by,
            criterion_name="-".join(criteria_),
            **kwargs,
        )
Source code for langchain.evaluation.qa.eval_chain
"""LLM Chains for evaluating question answering."""
from __future__ import annotations
import re
import string
from typing import Any, List, Optional, Sequence, Tuple
from langchain_core.callbacks.manager import Callbacks
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import PromptTemplate
from langchain_core.pydantic_v1 import Extra
from langchain.chains.llm import LLMChain
from langchain.evaluation.qa.eval_prompt import CONTEXT_PROMPT, COT_PROMPT, PROMPT
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.schema import RUN_KEY
def _get_score(text: str) -> Optional[Tuple[str, int]]:
    match = re.search(r"grade:\s*(correct|incorrect)", text.strip(), re.IGNORECASE)
    if match:
        if match.group(1).upper() == "CORRECT":
            return "CORRECT", 1
        elif match.group(1).upper() == "INCORRECT":
            return "INCORRECT", 0
    try:
        first_word = (
            text.strip().split()[0].translate(str.maketrans("", "", string.punctuation))
        )
        if first_word.upper() == "CORRECT":
            return "CORRECT", 1
        elif first_word.upper() == "INCORRECT":
            return "INCORRECT", 0
        last_word = (
            text.strip()
            .split()[-1]
            .translate(str.maketrans("", "", string.punctuation))
        )
        if last_word.upper() == "CORRECT":
            return "CORRECT", 1
        elif last_word.upper() == "INCORRECT":
Source code for langchain.evaluation.criteria.eval_chain
from __future__ import annotations
import re
from enum import Enum
from typing import Any, Dict, List, Mapping, Optional, Union
from langchain_core.callbacks.manager import Callbacks
from langchain_core.language_models import BaseLanguageModel
from langchain_core.output_parsers import BaseOutputParser
from langchain_core.prompts import BasePromptTemplate
from langchain_core.pydantic_v1 import Extra, Field
from langchain.chains.constitutional_ai.models import ConstitutionalPrinciple
from langchain.chains.llm import LLMChain
from langchain.evaluation.criteria.prompt import PROMPT, PROMPT_WITH_REFERENCES
from langchain.evaluation.schema import LLMEvalChain, StringEvaluator
from langchain.schema import RUN_KEY
[docs]class Criteria(str, Enum):
    """A Criteria to evaluate."""
    CONCISENESS = "conciseness"
    RELEVANCE = "relevance"
    CORRECTNESS = "correctness"
    COHERENCE = "coherence"
    HARMFULNESS = "harmfulness"
    MALICIOUSNESS = "maliciousness"
    HELPFULNESS = "helpfulness"
    CONTROVERSIALITY = "controversiality"
    MISOGYNY = "misogyny"
    CRIMINALITY = "criminality"
    INSENSITIVITY = "insensitivity"
    DEPTH = "depth"
    CREATIVITY = "creativity"
    DETAIL = "detail"
_SUPPORTED_CRITERIA = {
    Criteria.CONCISENESS: "Is the submission concise and to the point?",
    Criteria.RELEVANCE: "Is the submission referring to a real quote from the text?",
    Criteria.CORRECTNESS: "Is the submission correct, accurate, and factual?",
) -> Dict[str, str]:
    """Resolve the criteria to evaluate.
    Parameters
    ----------
    criteria : CRITERIA_TYPE
        The criteria to evaluate the runs against. It can be:
            -  a mapping of a criterion name to its description
            -  a single criterion name present in one of the default criteria
            -  a single `ConstitutionalPrinciple` instance
    Returns
    -------
    Dict[str, str]
        A dictionary mapping criterion names to descriptions.
    Examples
    --------
    >>> criterion = "relevance"
    >>> CriteriaEvalChain.resolve_criteria(criteria)
    {'relevance': 'Is the submission referring to a real quote from the text?'}
    """
    if criteria is None:
        return {
            "helpfulness": _SUPPORTED_CRITERIA[Criteria.HELPFULNESS],
        }
    if isinstance(criteria, Criteria):
        criteria_ = {criteria.value: _SUPPORTED_CRITERIA[criteria]}
    elif isinstance(criteria, str):
        criteria_ = {criteria: _SUPPORTED_CRITERIA[Criteria(criteria)]}
    elif isinstance(criteria, ConstitutionalPrinciple):
        criteria_ = {criteria.name: criteria.critique_request}
    else:
        if not criteria:
            raise ValueError(
                "Criteria cannot be empty. "
                "Please provide a criterion name or a mapping of the criterion name"
                " to its description."
            )
        criteria_ = dict(criteria)
    return criteria_
[docs]class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
    """LLM Chain for evaluating runs against criteria.
    Parameters
    ----------
    llm : BaseLanguageModel
        The language model to use for evaluation.
    criteria : Union[Mapping[str, str]]
criteria : Union[Mapping[str, str]]
        The criteria or rubric to evaluate the runs against. It can be a mapping of
        criterion name to its description, or a single criterion name.
    prompt : Optional[BasePromptTemplate], default=None
        The prompt template to use for generating prompts. If not provided, a
        default prompt template will be used based on the value of
        `requires_reference`.
    requires_reference : bool, default=False
        Whether the evaluation requires a reference text. If `True`, the
        `PROMPT_WITH_REFERENCES` template will be used, which includes the
        reference labels in the prompt. Otherwise, the `PROMPT` template will be
        used, which is a reference-free prompt.
    **kwargs : Any
        Additional keyword arguments to pass to the `LLMChain` constructor.
    Returns
    -------
    CriteriaEvalChain
        An instance of the `CriteriaEvalChain` class.
    Examples
    --------
    >>> from langchain_anthropic import ChatAnthropic
    >>> from langchain.evaluation.criteria import CriteriaEvalChain
    >>> llm = ChatAnthropic(temperature=0)
    >>> criteria = {"my-custom-criterion": "Is the submission the most amazing ever?"}
    >>> evaluator = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
    >>> evaluator.evaluate_strings(prediction="Imagine an ice cream flavor for the color aquamarine", input="Tell me an idea")
    {
{
        'reasoning': 'Here is my step-by-step reasoning for the given criteria:\\n\\nThe criterion is: "Is the submission the most amazing ever?" This is a subjective criterion and open to interpretation. The submission suggests an aquamarine-colored ice cream flavor which is creative but may or may not be considered the most amazing idea ever conceived. There are many possible amazing ideas and this one ice cream flavor suggestion may or may not rise to that level for every person. \\n\\nN',
        'value': 'N',
        'score': 0,
    }
    >>> from langchain_openai import ChatOpenAI
    >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
    >>> llm = ChatOpenAI(model="gpt-4", temperature=0)
    >>> criteria = "correctness"
    >>> evaluator = LabeledCriteriaEvalChain.from_llm(
    ...     llm=llm,
    ...     criteria=criteria,
    ... )
    >>> evaluator.evaluate_strings(
    ...   prediction="The answer is 4",
    ...   input="How many apples are there?",
    ...   reference="There are 3 apples",
    ...   )
    {
        'score': 0,
        'reasoning': 'The criterion for this task is the correctness of the submission. The submission states that there are 4 apples, but the reference indicates that there are actually 3 apples. Therefore, the submission is not correct, accurate, or factual according to the given criterion.\\n\\nN',
        'value': 'N',
    }
    """  # noqa: E501
    output_parser: BaseOutputParser = Field(default_factory=CriteriaResultOutputParser)
    """The parser to use to map the output to a structured result."""
[docs]    @classmethod
    def resolve_criteria(
        cls,
        criteria: Optional[Union[CRITERIA_TYPE, str]],
    ) -> Dict[str, str]:
        """Resolve the criteria to evaluate.
        Parameters
        ----------
        criteria : CRITERIA_TYPE
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        Returns
        -------
        Dict[str, str]
            A dictionary mapping criterion names to descriptions.
        Examples
        --------
        >>> criterion = "relevance"
        >>> CriteriaEvalChain.resolve_criteria(criteria)
        {'relevance': 'Is the submission referring to a real quote from the text?'}
        """
        return resolve_criteria(criteria)
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: Optional[CRITERIA_TYPE] = None,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `CriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt template will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        CriteriaEvalChain
            An instance of the `CriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain_openai import OpenAI
        >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
            )
        """
        prompt_ = cls._resolve_prompt(prompt)
        if criteria == Criteria.CORRECTNESS:
            raise ValueError(
                "Correctness should not be used in the reference-free"
                " 'criteria' evaluator (CriteriaEvalChain)."
                " Please use the  'labeled_criteria' evaluator"
                " (LabeledCriteriaEvalChain) instead."
            )
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = "\n".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt_.partial(criteria=criteria_str)
        return cls(
            llm=llm,
            prompt=prompt_,
            criterion_name="-".join(criteria_),
            **kwargs,
        )
    def _get_eval_input(
        self,
        prediction: str,
        reference: Optional[str],
        input: Optional[str],
prediction: str,
        reference: Optional[str],
        input: Optional[str],
    ) -> dict:
        """Get the evaluation input."""
        input_ = {
            "input": input,
            "output": prediction,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
    def _prepare_output(self, result: dict) -> dict:
        """Prepare the output."""
        parsed = result[self.output_key]
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        return parsed
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `__call__`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain_openai import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
>>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> chain.evaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = self(
            input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `acall`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain_openai import OpenAI
Examples
        --------
        >>> from langchain_openai import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> await chain.aevaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = await self.acall(
            input_,
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class LabeledCriteriaEvalChain(CriteriaEvalChain):
    """Criteria evaluation chain that requires references."""
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        """Whether the evaluation requires a reference text."""
        return True
    @classmethod
    def _resolve_prompt(
        cls, prompt: Optional[BasePromptTemplate] = None
    ) -> BasePromptTemplate:
        expected_input_vars = {"input", "output", "criteria", "reference"}
        prompt_ = prompt or PROMPT_WITH_REFERENCES
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        return prompt_
[docs]    @classmethod
    def from_llm(
        cls,
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: Optional[CRITERIA_TYPE] = None,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `LabeledCriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE - default=None for "helpfulness"
            The criteria to evaluate the runs against. It can be:
                -  a mapping of a criterion name to its description
                -  a single criterion name present in one of the default criteria
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        LabeledCriteriaEvalChain
            An instance of the `LabeledCriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain_openai import OpenAI
        >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = LabeledCriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
            )
        """
Source code for langchain.evaluation.parsing.json_distance
import json
from typing import Any, Callable, Optional, Union
from langchain_core.utils.json import parse_json_markdown
from langchain.evaluation.schema import StringEvaluator
[docs]class JsonEditDistanceEvaluator(StringEvaluator):
    """
    An evaluator that calculates the edit distance between JSON strings.
    This evaluator computes a normalized Damerau-Levenshtein distance between two JSON strings
    after parsing them and converting them to a canonical format (i.e., whitespace and key order are normalized).
    It can be customized with alternative distance and canonicalization functions.
    Args:
        string_distance (Optional[Callable[[str, str], float]]): A callable that computes the distance between two strings.
            If not provided, a Damerau-Levenshtein distance from the `rapidfuzz` package will be used.
        canonicalize (Optional[Callable[[Any], Any]]): A callable that converts a parsed JSON object into its canonical string form.
            If not provided, the default behavior is to serialize the JSON with sorted keys and no extra whitespace.
        **kwargs (Any): Additional keyword arguments.
    Attributes:
        _string_distance (Callable[[str, str], float]): The internal distance computation function.
        _canonicalize (Callable[[Any], Any]): The internal canonicalization function.
    Examples:
        >>> evaluator = JsonEditDistanceEvaluator()
        >>> result = evaluator.evaluate_strings(prediction='{"a": 1, "b": 2}', reference='{"a": 1, "b": 3}')
        >>> assert result["score"] is not None
    Raises:
        ImportError: If `rapidfuzz` is not installed and no alternative `string_distance` function is provided.
    """  # noqa: E501
[docs]    def __init__(
"""  # noqa: E501
[docs]    def __init__(
        self,
        string_distance: Optional[Callable[[str, str], float]] = None,
        canonicalize: Optional[Callable[[Any], Any]] = None,
        **kwargs: Any,
    ) -> None:
        super().__init__()
        if string_distance is not None:
            self._string_distance = string_distance
        else:
            try:
                from rapidfuzz import distance as rfd
            except ImportError:
                raise ImportError(
                    "The default string_distance operator for the "
                    " JsonEditDistanceEvaluator requires installation of "
                    "the rapidfuzz package. "
                    "Please install it with `pip install rapidfuzz`."
                )
            self._string_distance = rfd.DamerauLevenshtein.normalized_distance
        if canonicalize is not None:
            self._canonicalize = canonicalize
        else:
            self._canonicalize = lambda x: json.dumps(
                x,
                separators=(",", ":"),
                sort_keys=True,  # eliminate whitespace
            )
    @property
    def requires_input(self) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        return True
    @property
    def evaluation_name(self) -> str:
        return "json_edit_distance"
    def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
        if isinstance(node, str):
            return parse_json_markdown(node)
        return node
    def _evaluate_strings(
        self,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,