viruPVS/langchain-docs huggingface dataset:
"discussion",
    "disease",
    "disgust",
    "distance",
    "distribution",
    "division",
    "doctor",
    "dog",
    "door",
    "drain",
    "drawer",
    "dress",
    "drink",
    "driving",
    "dust",
    "ear",
    "earth",
    "edge",
    "education",
    "effect",
    "egg",
    "end",
    "energy",
    "engine",
    "error",
    "event",
    "example",
    "exchange",
    "existence",
    "expansion",
    "experience",
    "expert",
    "eye",
    "face",
    "fact",
    "fall",
    "family",
    "farm",
    "father",
    "fear",
    "feeling",
    "field",
    "finger",
    "fire",
    "fish",
    "flag",
    "flight",
    "floor",
    "flower",
    "fold",
    "food",
    "football",
    "force",
    "form",
    "frame",
    "friend",
    "frog",
    "fruit",
    "fuel",
    "furniture",
    "game",
    "garden",
    "gate",
    "girl",
    "glass",
    "glove",
    "goat",
    "gold",
    "government",
    "grade",
    "grain",
    "grass",
    "green",
    "grip",
    "group",
    "growth",
    "guide",
"group",
    "growth",
    "guide",
    "guitar",
    "hair",
    "hall",
    "hand",
    "harbor",
    "harmony",
    "hat",
    "head",
    "health",
    "heart",
    "heat",
    "hill",
    "history",
    "hobbies",
    "hole",
    "hope",
    "horn",
    "horse",
    "hospital",
    "hour",
    "house",
    "humor",
    "idea",
    "impulse",
    "income",
    "increase",
    "industry",
    "ink",
    "insect",
    "instrument",
    "insurance",
    "interest",
    "invention",
    "iron",
    "island",
    "jelly",
    "jet",
    "jewel",
    "join",
    "judge",
    "juice",
    "jump",
    "kettle",
    "key",
    "kick",
    "kiss",
    "kitten",
    "knee",
    "knife",
    "knowledge",
    "land",
    "language",
    "laugh",
    "law",
    "lead",
    "learning",
    "leather",
    "leg",
    "lettuce",
    "level",
    "library",
    "lift",
    "light",
    "limit",
    "line",
    "linen",
    "lip",
    "liquid",
    "list",
    "look",
    "loss",
    "love",
"list",
    "look",
    "loss",
    "love",
    "lunch",
    "machine",
    "man",
    "manager",
    "map",
    "marble",
    "mark",
    "market",
    "mass",
    "match",
    "meal",
    "measure",
    "meat",
    "meeting",
    "memory",
    "metal",
    "middle",
    "milk",
    "mind",
    "mine",
    "minute",
    "mist",
    "mitten",
    "mom",
    "money",
    "monkey",
    "month",
    "moon",
    "morning",
    "mother",
    "motion",
    "mountain",
    "mouth",
    "muscle",
    "music",
    "nail",
    "name",
    "nation",
    "neck",
    "need",
    "news",
    "night",
    "noise",
    "note",
    "number",
    "nut",
    "observation",
    "offer",
    "oil",
    "operation",
    "opinion",
    "orange",
    "order",
    "organization",
    "ornament",
    "oven",
    "page",
    "pail",
    "pain",
    "paint",
    "pan",
    "pancake",
    "paper",
    "parcel",
    "parent",
    "part",
    "passenger",
    "paste",
    "payment",
    "peace",
    "pear",
    "pen",
    "pencil",
"pear",
    "pen",
    "pencil",
    "person",
    "pest",
    "pet",
    "picture",
    "pie",
    "pin",
    "pipe",
    "pizza",
    "place",
    "plane",
    "plant",
    "plastic",
    "plate",
    "play",
    "pleasure",
    "plot",
    "plough",
    "pocket",
    "point",
    "poison",
    "police",
    "pollution",
    "popcorn",
    "porter",
    "position",
    "pot",
    "potato",
    "powder",
    "power",
    "price",
    "print",
    "process",
    "produce",
    "product",
    "profit",
    "property",
    "prose",
    "protest",
    "pull",
    "pump",
    "punishment",
    "purpose",
    "push",
    "quarter",
    "question",
    "quiet",
    "quill",
    "quilt",
    "quince",
    "rabbit",
    "rail",
    "rain",
    "range",
    "rat",
    "rate",
    "ray",
    "reaction",
    "reading",
    "reason",
    "record",
    "regret",
    "relation",
    "religion",
    "representative",
    "request",
    "respect",
    "rest",
    "reward",
    "rhythm",
    "rice",
    "river",
    "road",
"rice",
    "river",
    "road",
    "roll",
    "room",
    "root",
    "rose",
    "route",
    "rub",
    "rule",
    "run",
    "sack",
    "sail",
    "salt",
    "sand",
    "scale",
    "scarecrow",
    "scarf",
    "scene",
    "scent",
    "school",
    "science",
    "scissors",
    "screw",
    "sea",
    "seat",
    "secretary",
    "seed",
    "selection",
    "self",
    "sense",
    "servant",
    "shade",
    "shake",
    "shame",
    "shape",
    "sheep",
    "sheet",
    "shelf",
    "ship",
    "shirt",
    "shock",
    "shoe",
    "shop",
    "show",
    "side",
    "sign",
    "silk",
    "sink",
    "sister",
    "size",
    "sky",
    "sleep",
    "smash",
    "smell",
    "smile",
    "smoke",
    "snail",
    "snake",
    "sneeze",
    "snow",
    "soap",
    "society",
    "sock",
    "soda",
    "sofa",
    "son",
    "song",
    "sort",
    "sound",
    "soup",
    "space",
    "spark",
    "speed",
"soup",
    "space",
    "spark",
    "speed",
    "sponge",
    "spoon",
    "spray",
    "spring",
    "spy",
    "square",
    "stamp",
    "star",
    "start",
    "statement",
    "station",
    "steam",
    "steel",
    "stem",
    "step",
    "stew",
    "stick",
    "stitch",
    "stocking",
    "stomach",
    "stone",
    "stop",
    "store",
    "story",
    "stove",
    "stranger",
    "straw",
    "stream",
    "street",
    "stretch",
    "string",
    "structure",
    "substance",
    "sugar",
    "suggestion",
    "suit",
    "summer",
    "sun",
    "support",
    "surprise",
    "sweater",
    "swim",
    "system",
    "table",
    "tail",
    "talk",
    "tank",
    "taste",
    "tax",
    "tea",
    "teaching",
    "team",
    "tendency",
    "test",
    "texture",
    "theory",
    "thing",
    "thought",
    "thread",
    "throat",
    "thumb",
    "thunder",
    "ticket",
    "time",
    "tin",
    "title",
    "toad",
    "toe",
    "tooth",
    "toothpaste",
"toe",
    "tooth",
    "toothpaste",
    "touch",
    "town",
    "toy",
    "trade",
    "train",
    "transport",
    "tray",
    "treatment",
    "tree",
    "trick",
    "trip",
    "trouble",
    "trousers",
    "truck",
    "tub",
    "turkey",
    "turn",
    "twist",
    "umbrella",
    "uncle",
    "underwear",
    "unit",
    "use",
    "vacation",
    "value",
    "van",
    "vase",
    "vegetable",
    "veil",
    "vein",
    "verse",
    "vessel",
    "view",
    "visitor",
    "voice",
    "volcano",
    "walk",
    "wall",
    "war",
    "wash",
    "waste",
    "watch",
    "water",
    "wave",
    "wax",
    "way",
    "wealth",
    "weather",
    "week",
    "weight",
    "wheel",
    "whip",
    "whistle",
    "window",
    "wine",
    "wing",
    "winter",
    "wire",
    "wish",
    "woman",
    "wood",
    "wool",
    "word",
    "work",
    "worm",
    "wound",
    "wrist",
    "writer",
    "yard",
    "yoke",
"writer",
    "yard",
    "yoke",
    "zebra",
    "zinc",
    "zipper",
    "zone",
]
[docs]def random_name() -> str:
    """Generate a random name."""
    adjective = random.choice(adjectives)
    noun = random.choice(nouns)
    number = random.randint(1, 100)
    return f"{adjective}-{noun}-{number}"
Source code for langchain.smith.evaluation.progress
"""A simple progress bar for the console."""
import threading
from typing import Any, Dict, Optional, Sequence
from uuid import UUID
from langchain_core.documents import Document
from langchain_core.outputs import LLMResult
from langchain.callbacks import base as base_callbacks
[docs]class ProgressBarCallback(base_callbacks.BaseCallbackHandler):
    """A simple progress bar for the console."""
[docs]    def __init__(self, total: int, ncols: int = 50, **kwargs: Any):
        """Initialize the progress bar.
        Args:
            total: int, the total number of items to be processed.
            ncols: int, the character width of the progress bar.
        """
        self.total = total
        self.ncols = ncols
        self.counter = 0
        self.lock = threading.Lock()
        self._print_bar()
[docs]    def increment(self) -> None:
        """Increment the counter and update the progress bar."""
        with self.lock:
            self.counter += 1
            self._print_bar()
    def _print_bar(self) -> None:
        """Print the progress bar to the console."""
        progress = self.counter / self.total
        arrow = "-" * int(round(progress * self.ncols) - 1) + ">"
        spaces = " " * (self.ncols - len(arrow))
        print(f"\r[{arrow + spaces}] {self.counter}/{self.total}", end="")
[docs]    def on_chain_error(
        self,
        error: BaseException,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_chain_end(
        self,
        outputs: Dict[str, Any],
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_retriever_error(
        self,
        error: BaseException,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_retriever_end(
        self,
        documents: Sequence[Document],
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_llm_error(
        self,
        error: BaseException,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_llm_end(
        self,
        response: LLMResult,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_tool_error(
        self,
self.increment()
[docs]    def on_tool_error(
        self,
        error: BaseException,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
[docs]    def on_tool_end(
        self,
        output: str,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> Any:
        if parent_run_id is None:
            self.increment()
Source code for langchain.smith.evaluation.config
"""Configuration for run evaluators."""
from typing import Any, Dict, List, Optional, Union
from langchain_core.embeddings import Embeddings
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import BasePromptTemplate
from langchain_core.pydantic_v1 import BaseModel, Field
from langsmith import RunEvaluator
from langchain.evaluation.criteria.eval_chain import CRITERIA_TYPE
from langchain.evaluation.embedding_distance.base import (
    EmbeddingDistance as EmbeddingDistanceEnum,
)
from langchain.evaluation.schema import EvaluatorType, StringEvaluator
from langchain.evaluation.string_distance.base import (
    StringDistance as StringDistanceEnum,
)
[docs]class EvalConfig(BaseModel):
    """Configuration for a given run evaluator.
    Parameters
    ----------
    evaluator_type : EvaluatorType
        The type of evaluator to use.
    Methods
    -------
    get_kwargs()
        Get the keyword arguments for the evaluator configuration.
    """
    evaluator_type: EvaluatorType
[docs]    def get_kwargs(self) -> Dict[str, Any]:
        """Get the keyword arguments for the load_evaluator call.
        Returns
        -------
        Dict[str, Any]
            The keyword arguments for the load_evaluator call.
        """
        kwargs = {}
        for field, val in self:
            if field == "evaluator_type":
                continue
            elif val is None:
                continue
            kwargs[field] = val
        return kwargs
[docs]class SingleKeyEvalConfig(EvalConfig):
    reference_key: Optional[str] = None
    """The key in the dataset run to use as the reference string.
    If not provided, we will attempt to infer automatically."""
If not provided, we will attempt to infer automatically."""
    prediction_key: Optional[str] = None
    """The key from the traced run's outputs dictionary to use to
    represent the prediction. If not provided, it will be inferred
    automatically."""
    input_key: Optional[str] = None
    """The key from the traced run's inputs dictionary to use to represent the
    input. If not provided, it will be inferred automatically."""
[docs]    def get_kwargs(self) -> Dict[str, Any]:
        kwargs = super().get_kwargs()
        # Filer out the keys that are not needed for the evaluator.
        for key in ["reference_key", "prediction_key", "input_key"]:
            kwargs.pop(key, None)
        return kwargs
[docs]class RunEvalConfig(BaseModel):
    """Configuration for a run evaluation.
    Parameters
    ----------
    evaluators : List[Union[EvaluatorType, EvalConfig]]
        Configurations for which evaluators to apply to the dataset run.
        Each can be the string of an :class:`EvaluatorType <langchain.evaluation.schema.EvaluatorType>`, such
        as EvaluatorType.QA, the evaluator type string ("qa"), or a configuration for a
        given evaluator (e.g., :class:`RunEvalConfig.QA <langchain.smith.evaluation.config.RunEvalConfig.QA>`).
    custom_evaluators : Optional[List[Union[RunEvaluator, StringEvaluator]]]
        Custom evaluators to apply to the dataset run.
    reference_key : Optional[str]
        The key in the dataset run to use as the reference string.
        If not provided, it will be inferred automatically.
    prediction_key : Optional[str]
        The key from the traced run's outputs dictionary to use to
The key from the traced run's outputs dictionary to use to
        represent the prediction. If not provided, it will be inferred
        automatically.
    input_key : Optional[str]
        The key from the traced run's inputs dictionary to use to represent the
        input. If not provided, it will be inferred automatically.
    eval_llm : Optional[BaseLanguageModel]
        The language model to pass to any evaluators that use a language model.
    """  # noqa: E501
    evaluators: List[Union[EvaluatorType, str, EvalConfig]] = Field(
        default_factory=list
    )
    """Configurations for which evaluators to apply to the dataset run.
    Each can be the string of an
    :class:`EvaluatorType <langchain.evaluation.schema.EvaluatorType>`, such
    as `EvaluatorType.QA`, the evaluator type string ("qa"), or a configuration for a
    given evaluator
    (e.g., 
    :class:`RunEvalConfig.QA <langchain.smith.evaluation.config.RunEvalConfig.QA>`)."""  # noqa: E501
    custom_evaluators: Optional[List[Union[RunEvaluator, StringEvaluator]]] = None
    """Custom evaluators to apply to the dataset run."""
    reference_key: Optional[str] = None
    """The key in the dataset run to use as the reference string.
    If not provided, we will attempt to infer automatically."""
    prediction_key: Optional[str] = None
    """The key from the traced run's outputs dictionary to use to
    represent the prediction. If not provided, it will be inferred
    automatically."""
    input_key: Optional[str] = None
    """The key from the traced run's inputs dictionary to use to represent the
"""The key from the traced run's inputs dictionary to use to represent the
    input. If not provided, it will be inferred automatically."""
    eval_llm: Optional[BaseLanguageModel] = None
    """The language model to pass to any evaluators that require one."""
    class Config:
        arbitrary_types_allowed = True
[docs]    class Criteria(SingleKeyEvalConfig):
        """Configuration for a reference-free criteria evaluator.
        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        criteria: Optional[CRITERIA_TYPE] = None
        llm: Optional[BaseLanguageModel] = None
        evaluator_type: EvaluatorType = EvaluatorType.CRITERIA
        def __init__(
            self, criteria: Optional[CRITERIA_TYPE] = None, **kwargs: Any
        ) -> None:
            super().__init__(criteria=criteria, **kwargs)
[docs]    class LabeledCriteria(SingleKeyEvalConfig):
        """Configuration for a labeled (with references) criteria evaluator.
        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        criteria: Optional[CRITERIA_TYPE] = None
        llm: Optional[BaseLanguageModel] = None
        evaluator_type: EvaluatorType = EvaluatorType.LABELED_CRITERIA
        def __init__(
            self, criteria: Optional[CRITERIA_TYPE] = None, **kwargs: Any
        ) -> None:
) -> None:
            super().__init__(criteria=criteria, **kwargs)
[docs]    class EmbeddingDistance(SingleKeyEvalConfig):
        """Configuration for an embedding distance evaluator.
        Parameters
        ----------
        embeddings : Optional[Embeddings]
            The embeddings to use for computing the distance.
        distance_metric : Optional[EmbeddingDistanceEnum]
            The distance metric to use for computing the distance.
        """
        evaluator_type: EvaluatorType = EvaluatorType.EMBEDDING_DISTANCE
        embeddings: Optional[Embeddings] = None
        distance_metric: Optional[EmbeddingDistanceEnum] = None
        class Config:
            arbitrary_types_allowed = True
[docs]    class StringDistance(SingleKeyEvalConfig):
        """Configuration for a string distance evaluator.
        Parameters
        ----------
        distance : Optional[StringDistanceEnum]
            The string distance metric to use.
        """
        evaluator_type: EvaluatorType = EvaluatorType.STRING_DISTANCE
        distance: Optional[StringDistanceEnum] = None
        """The string distance metric to use.
            damerau_levenshtein: The Damerau-Levenshtein distance.
            levenshtein: The Levenshtein distance.
            jaro: The Jaro distance.
            jaro_winkler: The Jaro-Winkler distance.
        """
        normalize_score: bool = True
        """Whether to normalize the distance to between 0 and 1.
        Applies only to the Levenshtein and Damerau-Levenshtein distances."""
[docs]    class QA(SingleKeyEvalConfig):
        """Configuration for a QA evaluator.
        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        evaluator_type: EvaluatorType = EvaluatorType.QA
        llm: Optional[BaseLanguageModel] = None
        prompt: Optional[BasePromptTemplate] = None
[docs]    class ContextQA(SingleKeyEvalConfig):
        """Configuration for a context-based QA evaluator.
        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        evaluator_type: EvaluatorType = EvaluatorType.CONTEXT_QA
        llm: Optional[BaseLanguageModel] = None
        prompt: Optional[BasePromptTemplate] = None
[docs]    class CoTQA(SingleKeyEvalConfig):
        """Configuration for a context-based QA evaluator.
        Parameters
        ----------
        prompt : Optional[BasePromptTemplate]
            The prompt template to use for generating the question.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        """
        evaluator_type: EvaluatorType = EvaluatorType.CONTEXT_QA
        llm: Optional[BaseLanguageModel] = None
        prompt: Optional[BasePromptTemplate] = None
[docs]    class JsonValidity(SingleKeyEvalConfig):
        """Configuration for a json validity evaluator.
        Parameters
        ----------
        """
        evaluator_type: EvaluatorType = EvaluatorType.JSON_VALIDITY
[docs]    class JsonEqualityEvaluator(EvalConfig):
        """Configuration for a json equality evaluator.
        Parameters
        ----------
        """
"""Configuration for a json equality evaluator.
        Parameters
        ----------
        """
        evaluator_type: EvaluatorType = EvaluatorType.JSON_EQUALITY
[docs]    class ExactMatch(SingleKeyEvalConfig):
        """Configuration for an exact match string evaluator.
        Parameters
        ----------
        ignore_case : bool
            Whether to ignore case when comparing strings.
        ignore_punctuation : bool
            Whether to ignore punctuation when comparing strings.
        ignore_numbers : bool
            Whether to ignore numbers when comparing strings.
        """
        evaluator_type: EvaluatorType = EvaluatorType.STRING_DISTANCE
        ignore_case: bool = False
        ignore_punctuation: bool = False
        ignore_numbers: bool = False
[docs]    class RegexMatch(SingleKeyEvalConfig):
        """Configuration for a regex match string evaluator.
        Parameters
        ----------
        flags : int
            The flags to pass to the regex. Example: re.IGNORECASE.
        """
        evaluator_type: EvaluatorType = EvaluatorType.REGEX_MATCH
        flags: int = 0
[docs]    class ScoreString(SingleKeyEvalConfig):
        """Configuration for a score string evaluator.
        This is like the criteria evaluator but it is configured by
        default to return a score on the scale from 1-10.
        It is recommended to normalize these scores
        by setting `normalize_by` to 10.
        Parameters
        ----------
        criteria : Optional[CRITERIA_TYPE]
            The criteria to evaluate.
        llm : Optional[BaseLanguageModel]
            The language model to use for the evaluation chain.
        normalize_by: Optional[int] = None
            If you want to normalize the score, the denominator to use.
            If not provided, the score will be between 1 and 10 (by default).
If not provided, the score will be between 1 and 10 (by default).
        prompt : Optional[BasePromptTemplate]
        """
        evaluator_type: EvaluatorType = EvaluatorType.SCORE_STRING
        criteria: Optional[CRITERIA_TYPE] = None
        llm: Optional[BaseLanguageModel] = None
        normalize_by: Optional[float] = None
        prompt: Optional[BasePromptTemplate] = None
        def __init__(
            self,
            criteria: Optional[CRITERIA_TYPE] = None,
            normalize_by: Optional[float] = None,
            **kwargs: Any,
        ) -> None:
            super().__init__(criteria=criteria, normalize_by=normalize_by, **kwargs)
[docs]    class LabeledScoreString(ScoreString):
        evaluator_type: EvaluatorType = EvaluatorType.LABELED_SCORE_STRING