Tony20100/langchain_full huggingface dataset:
Source code for langchain_core.utils.json_schema
from __future__ import annotations
from copy import deepcopy
from typing import Any, Dict, List, Optional, Sequence, Set
def _retrieve_ref(path: str, schema: dict) -> dict:
    components = path.split("/")
    if components[0] != "#":
        raise ValueError(
            "ref paths are expected to be URI fragments, meaning they should start "
            "with #."
        )
    out = schema
    for component in components[1:]:
        if component.isdigit():
            out = out[int(component)]
        else:
            out = out[component]
    return deepcopy(out)
def _dereference_refs_helper(
    obj: Any,
    full_schema: Dict[str, Any],
    skip_keys: Sequence[str],
    processed_refs: Optional[Set[str]] = None,
) -> Any:
    if processed_refs is None:
        processed_refs = set()
    if isinstance(obj, dict):
        obj_out = {}
        for k, v in obj.items():
            if k in skip_keys:
                obj_out[k] = v
            elif k == "$ref":
                if v in processed_refs:
                    continue
                processed_refs.add(v)
                ref = _retrieve_ref(v, full_schema)
                full_ref = _dereference_refs_helper(
                    ref, full_schema, skip_keys, processed_refs
                )
                processed_refs.remove(v)
                return full_ref
            elif isinstance(v, (list, dict)):
                obj_out[k] = _dereference_refs_helper(
                    v, full_schema, skip_keys, processed_refs
                )
            else:
                obj_out[k] = v
        return obj_out
    elif isinstance(obj, list):
        return [
return obj_out
    elif isinstance(obj, list):
        return [
            _dereference_refs_helper(el, full_schema, skip_keys, processed_refs)
            for el in obj
        ]
    else:
        return obj
def _infer_skip_keys(
    obj: Any, full_schema: dict, processed_refs: Optional[Set[str]] = None
) -> List[str]:
    if processed_refs is None:
        processed_refs = set()
    keys = []
    if isinstance(obj, dict):
        for k, v in obj.items():
            if k == "$ref":
                if v in processed_refs:
                    continue
                processed_refs.add(v)
                ref = _retrieve_ref(v, full_schema)
                keys.append(v.split("/")[1])
                keys += _infer_skip_keys(ref, full_schema, processed_refs)
            elif isinstance(v, (list, dict)):
                keys += _infer_skip_keys(v, full_schema, processed_refs)
    elif isinstance(obj, list):
        for el in obj:
            keys += _infer_skip_keys(el, full_schema, processed_refs)
    return keys
[docs]def dereference_refs(
    schema_obj: dict,
    *,
    full_schema: Optional[dict] = None,
    skip_keys: Optional[Sequence[str]] = None,
) -> dict:
    """Try to substitute $refs in JSON Schema."""
    full_schema = full_schema or schema_obj
    skip_keys = (
        skip_keys
        if skip_keys is not None
        else _infer_skip_keys(schema_obj, full_schema)
    )
    return _dereference_refs_helper(schema_obj, full_schema, skip_keys)
Source code for langchain_core.utils.formatting
"""Utilities for formatting strings."""
from string import Formatter
from typing import Any, List, Mapping, Sequence
[docs]class StrictFormatter(Formatter):
    """Formatter that checks for extra keys."""
[docs]    def vformat(
        self, format_string: str, args: Sequence, kwargs: Mapping[str, Any]
    ) -> str:
        """Check that no arguments are provided."""
        if len(args) > 0:
            raise ValueError(
                "No arguments should be provided, "
                "everything should be passed as keyword arguments."
            )
        return super().vformat(format_string, args, kwargs)
[docs]    def validate_input_variables(
        self, format_string: str, input_variables: List[str]
    ) -> None:
        dummy_inputs = {input_variable: "foo" for input_variable in input_variables}
        super().format(format_string, **dummy_inputs)
formatter = StrictFormatter()
Source code for langchain_core.beta.runnables.context
import asyncio
import threading
from collections import defaultdict
from functools import partial
from itertools import groupby
from typing import (
    Any,
    Awaitable,
    Callable,
    DefaultDict,
    Dict,
    List,
    Mapping,
    Optional,
    Sequence,
    Type,
    TypeVar,
    Union,
)
from langchain_core._api.beta_decorator import beta
from langchain_core.runnables.base import (
    Runnable,
    RunnableSerializable,
    coerce_to_runnable,
)
from langchain_core.runnables.config import RunnableConfig, ensure_config, patch_config
from langchain_core.runnables.utils import ConfigurableFieldSpec, Input, Output
T = TypeVar("T")
Values = Dict[Union[asyncio.Event, threading.Event], Any]
CONTEXT_CONFIG_PREFIX = "__context__/"
CONTEXT_CONFIG_SUFFIX_GET = "/get"
CONTEXT_CONFIG_SUFFIX_SET = "/set"
async def _asetter(done: asyncio.Event, values: Values, value: T) -> T:
    values[done] = value
    done.set()
    return value
async def _agetter(done: asyncio.Event, values: Values) -> Any:
    await done.wait()
    return values[done]
def _setter(done: threading.Event, values: Values, value: T) -> T:
    values[done] = value
    done.set()
    return value
def _getter(done: threading.Event, values: Values) -> Any:
    done.wait()
    return values[done]
def _key_from_id(id_: str) -> str:
    wout_prefix = id_.split(CONTEXT_CONFIG_PREFIX, maxsplit=1)[1]
    if wout_prefix.endswith(CONTEXT_CONFIG_SUFFIX_GET):
if wout_prefix.endswith(CONTEXT_CONFIG_SUFFIX_GET):
        return wout_prefix[: -len(CONTEXT_CONFIG_SUFFIX_GET)]
    elif wout_prefix.endswith(CONTEXT_CONFIG_SUFFIX_SET):
        return wout_prefix[: -len(CONTEXT_CONFIG_SUFFIX_SET)]
    else:
        raise ValueError(f"Invalid context config id {id_}")
def _config_with_context(
    config: RunnableConfig,
    steps: List[Runnable],
    setter: Callable,
    getter: Callable,
    event_cls: Union[Type[threading.Event], Type[asyncio.Event]],
) -> RunnableConfig:
    if any(k.startswith(CONTEXT_CONFIG_PREFIX) for k in config.get("configurable", {})):
        return config
    context_specs = [
        (spec, i)
        for i, step in enumerate(steps)
        for spec in step.config_specs
        if spec.id.startswith(CONTEXT_CONFIG_PREFIX)
    ]
    grouped_by_key = {
        key: list(group)
        for key, group in groupby(
            sorted(context_specs, key=lambda s: s[0].id),
            key=lambda s: _key_from_id(s[0].id),
        )
    }
    deps_by_key = {
        key: set(
            _key_from_id(dep) for spec in group for dep in (spec[0].dependencies or [])
        )
        for key, group in grouped_by_key.items()
    }
    values: Values = {}
    events: DefaultDict[str, Union[asyncio.Event, threading.Event]] = defaultdict(
        event_cls
    )
    context_funcs: Dict[str, Callable[[], Any]] = {}
    for key, group in grouped_by_key.items():
for key, group in grouped_by_key.items():
        getters = [s for s in group if s[0].id.endswith(CONTEXT_CONFIG_SUFFIX_GET)]
        setters = [s for s in group if s[0].id.endswith(CONTEXT_CONFIG_SUFFIX_SET)]
        for dep in deps_by_key[key]:
            if key in deps_by_key[dep]:
                raise ValueError(
                    f"Deadlock detected between context keys {key} and {dep}"
                )
        if len(setters) != 1:
            raise ValueError(f"Expected exactly one setter for context key {key}")
        setter_idx = setters[0][1]
        if any(getter_idx < setter_idx for _, getter_idx in getters):
            raise ValueError(
                f"Context setter for key {key} must be defined after all getters."
            )
        if getters:
            context_funcs[getters[0][0].id] = partial(getter, events[key], values)
        context_funcs[setters[0][0].id] = partial(setter, events[key], values)
    return patch_config(config, configurable=context_funcs)
[docs]def aconfig_with_context(
    config: RunnableConfig,
    steps: List[Runnable],
) -> RunnableConfig:
    """Asynchronously patch a runnable config with context getters and setters.
    Args:
        config: The runnable config.
        steps: The runnable steps.
    Returns:
        The patched runnable config.
    """
    return _config_with_context(config, steps, _asetter, _agetter, asyncio.Event)
[docs]def config_with_context(
    config: RunnableConfig,
    steps: List[Runnable],
) -> RunnableConfig:
    """Patch a runnable config with context getters and setters.
    Args:
"""Patch a runnable config with context getters and setters.
    Args:
        config: The runnable config.
        steps: The runnable steps.
    Returns:
        The patched runnable config.
    """
    return _config_with_context(config, steps, _setter, _getter, threading.Event)
[docs]@beta()
class ContextGet(RunnableSerializable):
    """Get a context value."""
    prefix: str = ""
    key: Union[str, List[str]]
    def __str__(self) -> str:
        return f"ContextGet({_print_keys(self.key)})"
    @property
    def ids(self) -> List[str]:
        prefix = self.prefix + "/" if self.prefix else ""
        keys = self.key if isinstance(self.key, list) else [self.key]
        return [
            f"{CONTEXT_CONFIG_PREFIX}{prefix}{k}{CONTEXT_CONFIG_SUFFIX_GET}"
            for k in keys
        ]
    @property
    def config_specs(self) -> List[ConfigurableFieldSpec]:
        return super().config_specs + [
            ConfigurableFieldSpec(
                id=id_,
                annotation=Callable[[], Any],
            )
            for id_ in self.ids
        ]
[docs]    def invoke(self, input: Any, config: Optional[RunnableConfig] = None) -> Any:
        config = ensure_config(config)
        configurable = config.get("configurable", {})
        if isinstance(self.key, list):
            return {key: configurable[id_]() for key, id_ in zip(self.key, self.ids)}
        else:
            return configurable[self.ids[0]]()
[docs]    async def ainvoke(
        self, input: Any, config: Optional[RunnableConfig] = None, **kwargs: Any
    ) -> Any:
) -> Any:
        config = ensure_config(config)
        configurable = config.get("configurable", {})
        if isinstance(self.key, list):
            values = await asyncio.gather(*(configurable[id_]() for id_ in self.ids))
            return {key: value for key, value in zip(self.key, values)}
        else:
            return await configurable[self.ids[0]]()
SetValue = Union[
    Runnable[Input, Output],
    Callable[[Input], Output],
    Callable[[Input], Awaitable[Output]],
    Any,
]
def _coerce_set_value(value: SetValue) -> Runnable[Input, Output]:
    if not isinstance(value, Runnable) and not callable(value):
        return coerce_to_runnable(lambda _: value)
    return coerce_to_runnable(value)
[docs]@beta()
class ContextSet(RunnableSerializable):
    """Set a context value."""
    prefix: str = ""
    keys: Mapping[str, Optional[Runnable]]
    class Config:
        arbitrary_types_allowed = True
    def __init__(
        self,
        key: Optional[str] = None,
        value: Optional[SetValue] = None,
        prefix: str = "",
        **kwargs: SetValue,
    ):
        if key is not None:
            kwargs[key] = value
        super().__init__(  # type: ignore[call-arg]
            keys={
                k: _coerce_set_value(v) if v is not None else None
                for k, v in kwargs.items()
            },
            prefix=prefix,
        )
    def __str__(self) -> str:
        return f"ContextSet({_print_keys(list(self.keys.keys()))})"
    @property
    def ids(self) -> List[str]:
@property
    def ids(self) -> List[str]:
        prefix = self.prefix + "/" if self.prefix else ""
        return [
            f"{CONTEXT_CONFIG_PREFIX}{prefix}{key}{CONTEXT_CONFIG_SUFFIX_SET}"
            for key in self.keys
        ]
    @property
    def config_specs(self) -> List[ConfigurableFieldSpec]:
        mapper_config_specs = [
            s
            for mapper in self.keys.values()
            if mapper is not None
            for s in mapper.config_specs
        ]
        for spec in mapper_config_specs:
            if spec.id.endswith(CONTEXT_CONFIG_SUFFIX_GET):
                getter_key = spec.id.split("/")[1]
                if getter_key in self.keys:
                    raise ValueError(
                        f"Circular reference in context setter for key {getter_key}"
                    )
        return super().config_specs + [
            ConfigurableFieldSpec(
                id=id_,
                annotation=Callable[[], Any],
            )
            for id_ in self.ids
        ]
[docs]    def invoke(self, input: Any, config: Optional[RunnableConfig] = None) -> Any:
        config = ensure_config(config)
        configurable = config.get("configurable", {})
        for id_, mapper in zip(self.ids, self.keys.values()):
            if mapper is not None:
                configurable[id_](mapper.invoke(input, config))
            else:
                configurable[id_](input)
        return input
[docs]    async def ainvoke(
        self, input: Any, config: Optional[RunnableConfig] = None, **kwargs: Any
    ) -> Any:
        config = ensure_config(config)
        configurable = config.get("configurable", {})
        for id_, mapper in zip(self.ids, self.keys.values()):
            if mapper is not None:
if mapper is not None:
                await configurable[id_](await mapper.ainvoke(input, config))
            else:
                await configurable[id_](input)
        return input
[docs]class Context:
    """
    Context for a runnable.
    The `Context` class provides methods for creating context scopes,
    getters, and setters within a runnable. It allows for managing
    and accessing contextual information throughout the execution
    of a program.
    Example:
        .. code-block:: python
            from langchain_core.beta.runnables.context import Context
            from langchain_core.runnables.passthrough import RunnablePassthrough
            from langchain_core.prompts.prompt import PromptTemplate
            from langchain_core.output_parsers.string import StrOutputParser
            from tests.unit_tests.fake.llm import FakeListLLM
            chain = (
                Context.setter("input")
                | {
                    "context": RunnablePassthrough()
                            | Context.setter("context"),
                    "question": RunnablePassthrough(),
                }
                | PromptTemplate.from_template("{context} {question}")
                | FakeListLLM(responses=["hello"])
                | StrOutputParser()
                | {
                    "result": RunnablePassthrough(),
                    "context": Context.getter("context"),
                    "input": Context.getter("input"),
                }
            )
            # Use the chain
            output = chain.invoke("What's your name?")
            print(output["result"])  # Output: "hello"
            print(output["context"])  # Output: "What's your name?"
            print(output["input"])  # Output: "What's your name?
    """
[docs]    @staticmethod
    def create_scope(scope: str, /) -> "PrefixContext":
        """Create a context scope.
"""Create a context scope.
        Args:
            scope: The scope.
        Returns:
            The context scope.
        """
        return PrefixContext(prefix=scope)
[docs]    @staticmethod
    def getter(key: Union[str, List[str]], /) -> ContextGet:
        return ContextGet(key=key)
[docs]    @staticmethod
    def setter(
        _key: Optional[str] = None,
        _value: Optional[SetValue] = None,
        /,
        **kwargs: SetValue,
    ) -> ContextSet:
        return ContextSet(_key, _value, prefix="", **kwargs)
[docs]class PrefixContext:
    """Context for a runnable with a prefix."""
    prefix: str = ""
[docs]    def __init__(self, prefix: str = ""):
        self.prefix = prefix
[docs]    def getter(self, key: Union[str, List[str]], /) -> ContextGet:
        return ContextGet(key=key, prefix=self.prefix)
[docs]    def setter(
        self,
        _key: Optional[str] = None,
        _value: Optional[SetValue] = None,
        /,
        **kwargs: SetValue,
    ) -> ContextSet:
        return ContextSet(_key, _value, prefix=self.prefix, **kwargs)
def _print_keys(keys: Union[str, Sequence[str]]) -> str:
    if isinstance(keys, str):
        return f"'{keys}'"
    else:
        return ", ".join(f"'{k}'" for k in keys)
Source code for langchain_core.output_parsers.openai_tools
import copy
import json
from json import JSONDecodeError
from typing import Any, Dict, List, Optional, Type
from langchain_core.exceptions import OutputParserException
from langchain_core.messages import AIMessage, InvalidToolCall
from langchain_core.output_parsers import BaseCumulativeTransformOutputParser
from langchain_core.outputs import ChatGeneration, Generation
from langchain_core.pydantic_v1 import BaseModel, ValidationError
from langchain_core.utils.json import parse_partial_json
[docs]def parse_tool_call(
    raw_tool_call: Dict[str, Any],
    *,
    partial: bool = False,
    strict: bool = False,
    return_id: bool = True,
) -> Optional[Dict[str, Any]]:
    """Parse a single tool call."""
    if "function" not in raw_tool_call:
        return None
    if partial:
        try:
            function_args = parse_partial_json(
                raw_tool_call["function"]["arguments"], strict=strict
            )
        except (JSONDecodeError, TypeError):  # None args raise TypeError
            return None
    else:
        try:
            function_args = json.loads(
                raw_tool_call["function"]["arguments"], strict=strict
            )
        except JSONDecodeError as e:
            raise OutputParserException(
                f"Function {raw_tool_call['function']['name']} arguments:\n\n"
                f"{raw_tool_call['function']['arguments']}\n\nare not valid JSON. "
                f"Received JSONDecodeError {e}"
            )
    parsed = {
        "name": raw_tool_call["function"]["name"] or "",
        "args": function_args or {},
    }
    if return_id:
"args": function_args or {},
    }
    if return_id:
        parsed["id"] = raw_tool_call.get("id")
    return parsed
[docs]def make_invalid_tool_call(
    raw_tool_call: Dict[str, Any],
    error_msg: Optional[str],
) -> InvalidToolCall:
    """Create an InvalidToolCall from a raw tool call."""
    return InvalidToolCall(
        name=raw_tool_call["function"]["name"],
        args=raw_tool_call["function"]["arguments"],
        id=raw_tool_call.get("id"),
        error=error_msg,
    )
[docs]def parse_tool_calls(
    raw_tool_calls: List[dict],
    *,
    partial: bool = False,
    strict: bool = False,
    return_id: bool = True,
) -> List[Dict[str, Any]]:
    """Parse a list of tool calls."""
    final_tools: List[Dict[str, Any]] = []
    exceptions = []
    for tool_call in raw_tool_calls:
        try:
            parsed = parse_tool_call(
                tool_call, partial=partial, strict=strict, return_id=return_id
            )
            if parsed:
                final_tools.append(parsed)
        except OutputParserException as e:
            exceptions.append(str(e))
            continue
    if exceptions:
        raise OutputParserException("\n\n".join(exceptions))
    return final_tools
[docs]class JsonOutputToolsParser(BaseCumulativeTransformOutputParser[Any]):
    """Parse tools from OpenAI response."""
    strict: bool = False
    """Whether to allow non-JSON-compliant strings.
    See: https://docs.python.org/3/library/json.html#encoders-and-decoders
Useful when the parsed output may include unicode characters or new lines.
    """
    return_id: bool = False
    """Whether to return the tool call id."""
    first_tool_only: bool = False
    """Whether to return only the first tool call.
    If False, the result will be a list of tool calls, or an empty list 
    if no tool calls are found.
    If true, and multiple tool calls are found, only the first one will be returned,
    and the other tool calls will be ignored. 
    If no tool calls are found, None will be returned. 
    """
[docs]    def parse_result(self, result: List[Generation], *, partial: bool = False) -> Any:
        generation = result[0]
        if not isinstance(generation, ChatGeneration):
            raise OutputParserException(
                "This output parser can only be used with a chat generation."
            )
        message = generation.message
        if isinstance(message, AIMessage) and message.tool_calls:
            tool_calls = [dict(tc) for tc in message.tool_calls]
            for tool_call in tool_calls:
                if not self.return_id:
                    _ = tool_call.pop("id")
        else:
            try:
                raw_tool_calls = copy.deepcopy(message.additional_kwargs["tool_calls"])
            except KeyError:
                return []
            tool_calls = parse_tool_calls(
                raw_tool_calls,
                partial=partial,
                strict=self.strict,
                return_id=self.return_id,
            )
        # for backwards compatibility
        for tc in tool_calls:
            tc["type"] = tc.pop("name")
        if self.first_tool_only:
            return tool_calls[0] if tool_calls else None
        return tool_calls
return tool_calls[0] if tool_calls else None
        return tool_calls
[docs]    def parse(self, text: str) -> Any:
        raise NotImplementedError()
[docs]class JsonOutputKeyToolsParser(JsonOutputToolsParser):
    """Parse tools from OpenAI response."""
    key_name: str
    """The type of tools to return."""
[docs]    def parse_result(self, result: List[Generation], *, partial: bool = False) -> Any:
        parsed_result = super().parse_result(result, partial=partial)
        if self.first_tool_only:
            single_result = (
                parsed_result
                if parsed_result and parsed_result["type"] == self.key_name
                else None
            )
            if self.return_id:
                return single_result
            elif single_result:
                return single_result["args"]
            else:
                return None
        parsed_result = [res for res in parsed_result if res["type"] == self.key_name]
        if not self.return_id:
            parsed_result = [res["args"] for res in parsed_result]
        return parsed_result
[docs]class PydanticToolsParser(JsonOutputToolsParser):
    """Parse tools from OpenAI response."""
    tools: List[Type[BaseModel]]
    # TODO: Support more granular streaming of objects. Currently only streams once all
    # Pydantic object fields are present.
[docs]    def parse_result(self, result: List[Generation], *, partial: bool = False) -> Any:
        json_results = super().parse_result(result, partial=partial)
        if not json_results:
            return None if self.first_tool_only else []
        json_results = [json_results] if self.first_tool_only else json_results
json_results = [json_results] if self.first_tool_only else json_results
        name_dict = {tool.__name__: tool for tool in self.tools}
        pydantic_objects = []
        for res in json_results:
            try:
                if not isinstance(res["args"], dict):
                    raise ValueError(
                        f"Tool arguments must be specified as a dict, received: "
                        f"{res['args']}"
                    )
                pydantic_objects.append(name_dict[res["type"]](**res["args"]))
            except (ValidationError, ValueError) as e:
                if partial:
                    continue
                else:
                    raise e
        if self.first_tool_only:
            return pydantic_objects[0] if pydantic_objects else None
        else:
            return pydantic_objects