viruPVS/langchain-docs huggingface dataset:
)
    @property
    def requires_input(self) -> bool:
        """Returns whether the evaluator requires input."""
        return False
    @property
    def requires_reference(self) -> bool:
        """Returns whether the evaluator requires reference."""
        return True
    @property
    def evaluation_name(self) -> str:
        """Returns the name of the evaluation."""
        return "json_schema_validation"
    def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
        if isinstance(node, str):
            return parse_json_markdown(node)
        elif hasattr(node, "schema") and callable(getattr(node, "schema")):
            # Pydantic model
            return getattr(node, "schema")()
        return node
    def _validate(self, prediction: Any, schema: Any) -> dict:
        from jsonschema import ValidationError, validate  # noqa: F401
        try:
            validate(instance=prediction, schema=schema)
            return {
                "score": True,
            }
        except ValidationError as e:
            return {"score": False, "reasoning": repr(e)}
    def _evaluate_strings(
        self,
        prediction: Union[str, Any],
        input: Union[str, Any] = None,
        reference: Union[str, Any] = None,
        **kwargs: Any,
    ) -> dict:
        parsed_prediction = self._parse_json(prediction)
        schema = self._parse_json(reference)
        return self._validate(parsed_prediction, schema)
Source code for langchain.evaluation.parsing.base
"""Evaluators for parsing strings."""
from operator import eq
from typing import Any, Callable, Optional, Union, cast
from langchain.evaluation.schema import StringEvaluator
from langchain.output_parsers.json import parse_json_markdown
[docs]class JsonValidityEvaluator(StringEvaluator):
    """Evaluates whether the prediction is valid JSON.
    This evaluator checks if the prediction is a valid JSON string. It does not
        require any input or reference.
    Attributes:
        requires_input (bool): Whether this evaluator requires an input
            string. Always False.
        requires_reference (bool): Whether this evaluator requires a
            reference string. Always False.
        evaluation_name (str): The name of the evaluation metric.
            Always "json".
    Examples:
        >>> evaluator = JsonValidityEvaluator()
        >>> prediction = '{"name": "John", "age": 30, "city": "New York"}'
        >>> evaluator.evaluate(prediction)
        {'score': 1}
        >>> prediction = '{"name": "John", "age": 30, "city": "New York",}'
        >>> evaluator.evaluate(prediction)
        {'score': 0, 'reasoning': 'Expecting property name enclosed in double quotes'}
    """
[docs]    def __init__(self, **kwargs: Any) -> None:
        super().__init__()
    @property
    def requires_input(self) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        return False
    @property
    def evaluation_name(self) -> str:
        return "json_validity"
    def _evaluate_strings(
        self,
        prediction: str,
        input: Optional[str] = None,
self,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
        **kwargs: Any,
    ) -> dict:
        """Evaluate the prediction string.
        Args:
            prediction (str): The prediction string to evaluate.
            input (str, optional): Not used in this evaluator. Defaults to None.
            reference (str, optional): Not used in this evaluator. Defaults to None.
        Returns:
            dict: A dictionary containing the evaluation score. The score is 1 if
            the prediction is valid JSON, and 0 otherwise.
                If the prediction is not valid JSON, the dictionary also contains
                a "reasoning" field with the error message.
        """
        try:
            parse_json_markdown(prediction)
            return {"score": 1}
        except Exception as e:
            return {"score": 0, "reasoning": str(e)}
[docs]class JsonEqualityEvaluator(StringEvaluator):
    """Evaluates whether the prediction is equal to the reference after
        parsing both as JSON.
    This evaluator checks if the prediction, after parsing as JSON, is equal
        to the reference,
    which is also parsed as JSON. It does not require an input string.
    Attributes:
        requires_input (bool): Whether this evaluator requires an
            input string. Always False.
        requires_reference (bool): Whether this evaluator requires
            a reference string. Always True.
        evaluation_name (str): The name of the evaluation metric.
            Always "parsed_equality".
    Examples:
        >>> evaluator = JsonEqualityEvaluator()
        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 1}')
        {'score': True}
{'score': True}
        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 2}')
        {'score': False}
        >>> evaluator = JsonEqualityEvaluator(operator=lambda x, y: x['a'] == y['a'])
        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 1}')
        {'score': True}
        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 2}')
        {'score': False}
    """
[docs]    def __init__(self, operator: Optional[Callable] = None, **kwargs: Any) -> None:
        super().__init__()
        self.operator = operator or eq
    @property
    def requires_input(self) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        return True
    @property
    def evaluation_name(self) -> str:
        return "json_equality"
    def _parse_json(
        self,
        string: Any,
    ) -> Union[dict, list, None, float, bool, int, str]:
        if isinstance(string, str):
            return parse_json_markdown(string)
        return string
    def _evaluate_strings(
        self,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
        **kwargs: Any,
    ) -> dict:
        """Evaluate the prediction string.
        Args:
            prediction (str): The prediction string to evaluate.
            input (str, optional): Not used in this evaluator.
            reference (str): The reference string to compare against.
        Returns:
            dict: A dictionary containing the evaluation score.
        """
Returns:
            dict: A dictionary containing the evaluation score.
        """
        parsed = self._parse_json(prediction)
        label = self._parse_json(cast(str, reference))
        if isinstance(label, list):
            if not isinstance(parsed, list):
                return {"score": 0}
            parsed = sorted(parsed, key=lambda x: str(x))
            label = sorted(label, key=lambda x: str(x))
        return {"score": self.operator(parsed, label)}
Source code for langchain.evaluation.parsing.json_distance
import json
from typing import Any, Callable, Optional, Union
from langchain.evaluation.schema import StringEvaluator
from langchain.output_parsers.json import parse_json_markdown
[docs]class JsonEditDistanceEvaluator(StringEvaluator):
    """
    An evaluator that calculates the edit distance between JSON strings.
    This evaluator computes a normalized Damerau-Levenshtein distance between two JSON strings
    after parsing them and converting them to a canonical format (i.e., whitespace and key order are normalized).
    It can be customized with alternative distance and canonicalization functions.
    Args:
        string_distance (Optional[Callable[[str, str], float]]): A callable that computes the distance between two strings.
            If not provided, a Damerau-Levenshtein distance from the `rapidfuzz` package will be used.
        canonicalize (Optional[Callable[[Any], Any]]): A callable that converts a parsed JSON object into its canonical string form.
            If not provided, the default behavior is to serialize the JSON with sorted keys and no extra whitespace.
        **kwargs (Any): Additional keyword arguments.
    Attributes:
        _string_distance (Callable[[str, str], float]): The internal distance computation function.
        _canonicalize (Callable[[Any], Any]): The internal canonicalization function.
    Examples:
        >>> evaluator = JsonEditDistanceEvaluator()
        >>> result = evaluator.evaluate_strings(prediction='{"a": 1, "b": 2}', reference='{"a": 1, "b": 3}')
        >>> assert result["score"] is not None
    Raises:
        ImportError: If `rapidfuzz` is not installed and no alternative `string_distance` function is provided.
    """  # noqa: E501
[docs]    def __init__(
"""  # noqa: E501
[docs]    def __init__(
        self,
        string_distance: Optional[Callable[[str, str], float]] = None,
        canonicalize: Optional[Callable[[Any], Any]] = None,
        **kwargs: Any,
    ) -> None:
        super().__init__()
        if string_distance is not None:
            self._string_distance = string_distance
        else:
            try:
                from rapidfuzz import distance as rfd  # noqa: F401
            except ImportError:
                raise ImportError(
                    "The default string_distance operator for the "
                    " JsonEditDistanceEvaluator requires installation of "
                    "the rapidfuzz package. "
                    "Please install it with `pip install rapidfuzz`."
                )
            self._string_distance = rfd.DamerauLevenshtein.normalized_distance
        if canonicalize is not None:
            self._canonicalize = canonicalize
        else:
            self._canonicalize = lambda x: json.dumps(
                x,
                separators=(",", ":"),
                sort_keys=True,  # eliminate whitespace
            )
    @property
    def requires_input(self) -> bool:
        return False
    @property
    def requires_reference(self) -> bool:
        return True
    @property
    def evaluation_name(self) -> str:
        return "json_edit_distance"
    def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
        if isinstance(node, str):
            return parse_json_markdown(node)
        return node
    def _evaluate_strings(
        self,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
Source code for langchain.evaluation.embedding_distance.base
"""A chain for comparing the output of two models using embeddings."""
from enum import Enum
from typing import Any, Dict, List, Optional
import numpy as np
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import Field, root_validator
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
    Callbacks,
)
from langchain.chains.base import Chain
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.evaluation.schema import PairwiseStringEvaluator, StringEvaluator
from langchain.schema import RUN_KEY
from langchain.utils.math import cosine_similarity
[docs]class EmbeddingDistance(str, Enum):
    """Embedding Distance Metric.
    Attributes:
        COSINE: Cosine distance metric.
        EUCLIDEAN: Euclidean distance metric.
        MANHATTAN: Manhattan distance metric.
        CHEBYSHEV: Chebyshev distance metric.
        HAMMING: Hamming distance metric.
    """
    COSINE = "cosine"
    EUCLIDEAN = "euclidean"
    MANHATTAN = "manhattan"
    CHEBYSHEV = "chebyshev"
    HAMMING = "hamming"
class _EmbeddingDistanceChainMixin(Chain):
    """Shared functionality for embedding distance evaluators.
    Attributes:
        embeddings (Embeddings): The embedding objects to vectorize the outputs.
        distance_metric (EmbeddingDistance): The distance metric to use
                                            for comparing the embeddings.
    """
    embeddings: Embeddings = Field(default_factory=OpenAIEmbeddings)
    distance_metric: EmbeddingDistance = Field(default=EmbeddingDistance.COSINE)
distance_metric: EmbeddingDistance = Field(default=EmbeddingDistance.COSINE)
    @root_validator(pre=False)
    def _validate_tiktoken_installed(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate that the TikTok library is installed.
        Args:
            values (Dict[str, Any]): The values to validate.
        Returns:
            Dict[str, Any]: The validated values.
        """
        embeddings = values.get("embeddings")
        if isinstance(embeddings, OpenAIEmbeddings):
            try:
                import tiktoken  # noqa: F401
            except ImportError:
                raise ImportError(
                    "The tiktoken library is required to use the default "
                    "OpenAI embeddings with embedding distance evaluators."
                    " Please either manually select a different Embeddings object"
                    " or install tiktoken using `pip install tiktoken`."
                )
        return values
    class Config:
        """Permit embeddings to go unvalidated."""
        arbitrary_types_allowed: bool = True
    @property
    def output_keys(self) -> List[str]:
        """Return the output keys of the chain.
        Returns:
            List[str]: The output keys.
        """
        return ["score"]
    def _prepare_output(self, result: dict) -> dict:
        parsed = {"score": result["score"]}
        if RUN_KEY in result:
            parsed[RUN_KEY] = result[RUN_KEY]
        return parsed
    def _get_metric(self, metric: EmbeddingDistance) -> Any:
        """Get the metric function for the given metric name.
        Args:
            metric (EmbeddingDistance): The metric name.
        Returns:
            Any: The metric function.
        """
        metrics = {
Returns:
            Any: The metric function.
        """
        metrics = {
            EmbeddingDistance.COSINE: self._cosine_distance,
            EmbeddingDistance.EUCLIDEAN: self._euclidean_distance,
            EmbeddingDistance.MANHATTAN: self._manhattan_distance,
            EmbeddingDistance.CHEBYSHEV: self._chebyshev_distance,
            EmbeddingDistance.HAMMING: self._hamming_distance,
        }
        if metric in metrics:
            return metrics[metric]
        else:
            raise ValueError(f"Invalid metric: {metric}")
    @staticmethod
    def _cosine_distance(a: np.ndarray, b: np.ndarray) -> np.ndarray:
        """Compute the cosine distance between two vectors.
        Args:
            a (np.ndarray): The first vector.
            b (np.ndarray): The second vector.
        Returns:
            np.ndarray: The cosine distance.
        """
        return 1.0 - cosine_similarity(a, b)
    @staticmethod
    def _euclidean_distance(a: np.ndarray, b: np.ndarray) -> np.floating:
        """Compute the Euclidean distance between two vectors.
        Args:
            a (np.ndarray): The first vector.
            b (np.ndarray): The second vector.
        Returns:
            np.floating: The Euclidean distance.
        """
        return np.linalg.norm(a - b)
    @staticmethod
    def _manhattan_distance(a: np.ndarray, b: np.ndarray) -> np.floating:
        """Compute the Manhattan distance between two vectors.
        Args:
            a (np.ndarray): The first vector.
            b (np.ndarray): The second vector.
        Returns:
            np.floating: The Manhattan distance.
Returns:
            np.floating: The Manhattan distance.
        """
        return np.sum(np.abs(a - b))
    @staticmethod
    def _chebyshev_distance(a: np.ndarray, b: np.ndarray) -> np.floating:
        """Compute the Chebyshev distance between two vectors.
        Args:
            a (np.ndarray): The first vector.
            b (np.ndarray): The second vector.
        Returns:
            np.floating: The Chebyshev distance.
        """
        return np.max(np.abs(a - b))
    @staticmethod
    def _hamming_distance(a: np.ndarray, b: np.ndarray) -> np.floating:
        """Compute the Hamming distance between two vectors.
        Args:
            a (np.ndarray): The first vector.
            b (np.ndarray): The second vector.
        Returns:
            np.floating: The Hamming distance.
        """
        return np.mean(a != b)
    def _compute_score(self, vectors: np.ndarray) -> float:
        """Compute the score based on the distance metric.
        Args:
            vectors (np.ndarray): The input vectors.
        Returns:
            float: The computed score.
        """
        metric = self._get_metric(self.distance_metric)
        score = metric(vectors[0].reshape(1, -1), vectors[1].reshape(1, -1)).item()
        return score
[docs]class EmbeddingDistanceEvalChain(_EmbeddingDistanceChainMixin, StringEvaluator):
    """Use embedding distances to score semantic difference between
    a prediction and reference.
    Examples:
        >>> chain = EmbeddingDistanceEvalChain()
        >>> result = chain.evaluate_strings(prediction="Hello", reference="Hi")
        >>> print(result)
>>> print(result)
        {'score': 0.5}
    """
    @property
    def requires_reference(self) -> bool:
        """Return whether the chain requires a reference.
        Returns:
            bool: True if a reference is required, False otherwise.
        """
        return True
    @property
    def evaluation_name(self) -> str:
        return f"embedding_{self.distance_metric.value}_distance"
    @property
    def input_keys(self) -> List[str]:
        """Return the input keys of the chain.
        Returns:
            List[str]: The input keys.
        """
        return ["prediction", "reference"]
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Compute the score for a prediction and reference.
        Args:
            inputs (Dict[str, Any]): The input data.
            run_manager (Optional[CallbackManagerForChainRun], optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The computed score.
        """
        vectors = np.array(
            self.embeddings.embed_documents([inputs["prediction"], inputs["reference"]])
        )
        score = self._compute_score(vectors)
        return {"score": score}
    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Asynchronously compute the score for a prediction and reference.
        Args:
            inputs (Dict[str, Any]): The input data.
            run_manager (AsyncCallbackManagerForChainRun, optional):
run_manager (AsyncCallbackManagerForChainRun, optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The computed score.
        """
        embedded = await self.embeddings.aembed_documents(
            [inputs["prediction"], inputs["reference"]]
        )
        vectors = np.array(embedded)
        score = self._compute_score(vectors)
        return {"score": score}
    def _evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Evaluate the embedding distance between a prediction and
        reference.
        Args:
            prediction (str): The output string from the first model.
            reference (str): The reference string (required)
            callbacks (Callbacks, optional): The callbacks to use.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - score: The embedding distance between the two
                    predictions.
        """
        result = self(
            inputs={"prediction": prediction, "reference": reference},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
    async def _aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,
        metadata: Optional[Dict[str, Any]] = None,
        include_run_info: bool = False,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate the embedding distance between
        a prediction and reference.
        Args:
            prediction (str): The output string from the first model.
            reference (str): The output string from the second model.
            callbacks (Callbacks, optional): The callbacks to use.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - score: The embedding distance between the two
                    predictions.
        """
        result = await self.acall(
            inputs={"prediction": prediction, "reference": reference},
            callbacks=callbacks,
            tags=tags,
            metadata=metadata,
            include_run_info=include_run_info,
        )
        return self._prepare_output(result)
[docs]class PairwiseEmbeddingDistanceEvalChain(
    _EmbeddingDistanceChainMixin, PairwiseStringEvaluator
):
    """Use embedding distances to score semantic difference between two predictions.
    Examples:
    >>> chain = PairwiseEmbeddingDistanceEvalChain()
    >>> result = chain.evaluate_string_pairs(prediction="Hello", prediction_b="Hi")
    >>> print(result)
    {'score': 0.5}
    """
    @property
    def input_keys(self) -> List[str]:
        """Return the input keys of the chain.
        Returns:
            List[str]: The input keys.
        """
        return ["prediction", "prediction_b"]
    @property
    def evaluation_name(self) -> str:
        return f"pairwise_embedding_{self.distance_metric.value}_distance"
return f"pairwise_embedding_{self.distance_metric.value}_distance"
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Compute the score for two predictions.
        Args:
            inputs (Dict[str, Any]): The input data.
            run_manager (CallbackManagerForChainRun, optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The computed score.
        """
        vectors = np.array(
            self.embeddings.embed_documents(
                [inputs["prediction"], inputs["prediction_b"]]
            )
        )
        score = self._compute_score(vectors)
        return {"score": score}
    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Asynchronously compute the score for two predictions.
        Args:
            inputs (Dict[str, Any]): The input data.
            run_manager (AsyncCallbackManagerForChainRun, optional):
                The callback manager.
        Returns:
            Dict[str, Any]: The computed score.
        """
        embedded = await self.embeddings.aembed_documents(
            [inputs["prediction"], inputs["prediction_b"]]
        )
        vectors = np.array(embedded)
        score = self._compute_score(vectors)
        return {"score": score}
    def _evaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        callbacks: Callbacks = None,
        tags: Optional[List[str]] = None,