Tony20100/langchain_full huggingface dataset:
See documentation in XMLOutputParser for more information.
        """
        if parser == "defusedxml":
            try:
                from defusedxml import ElementTree as DET  # type: ignore
            except ImportError:
                raise ImportError(
                    "defusedxml is not installed. "
                    "Please install it to use the defusedxml parser."
                    "You can install it with `pip install defusedxml` "
                )
            _parser = DET.DefusedXMLParser(target=TreeBuilder())
        else:
            _parser = None
        self.pull_parser = ET.XMLPullParser(["start", "end"], _parser=_parser)
        self.xml_start_re = re.compile(r"<[a-zA-Z:_]")
        self.current_path: List[str] = []
        self.current_path_has_children = False
        self.buffer = ""
        self.xml_started = False
    def parse(self, chunk: Union[str, BaseMessage]) -> Iterator[AddableDict]:
        """Parse a chunk of text.
        Args:
            chunk: A chunk of text to parse. This can be a string or a BaseMessage.
        Yields:
            AddableDict: A dictionary representing the parsed XML element.
        """
        if isinstance(chunk, BaseMessage):
            # extract text
            chunk_content = chunk.content
            if not isinstance(chunk_content, str):
                # ignore non-string messages (e.g., function calls)
                return
            chunk = chunk_content
        # add chunk to buffer of unprocessed text
        self.buffer += chunk
        # if xml string hasn't started yet, continue to next chunk
        if not self.xml_started:
            if match := self.xml_start_re.search(self.buffer):
                # if xml string has started, remove all text before it
# if xml string has started, remove all text before it
                self.buffer = self.buffer[match.start() :]
                self.xml_started = True
            else:
                return
        # feed buffer to parser
        self.pull_parser.feed(self.buffer)
        self.buffer = ""
        # yield all events
        try:
            for event, elem in self.pull_parser.read_events():
                if event == "start":
                    # update current path
                    self.current_path.append(elem.tag)
                    self.current_path_has_children = False
                elif event == "end":
                    # remove last element from current path
                    #
                    self.current_path.pop()
                    # yield element
                    if not self.current_path_has_children:
                        yield nested_element(self.current_path, elem)
                    # prevent yielding of parent element
                    if self.current_path:
                        self.current_path_has_children = True
                    else:
                        self.xml_started = False
        except xml.etree.ElementTree.ParseError:
            # This might be junk at the end of the XML input.
            # Let's check whether the current path is empty.
            if not self.current_path:
                # If it is empty, we can ignore this error.
                return
            else:
                raise
    def close(self) -> None:
        """Close the parser."""
        try:
            self.pull_parser.close()
        except xml.etree.ElementTree.ParseError:
            # Ignore. This will ignore any incomplete XML at the end of the input
            pass
[docs]class XMLOutputParser(BaseTransformOutputParser):
    """Parse an output using xml format."""
    tags: Optional[List[str]] = None
    encoding_matcher: re.Pattern = re.compile(
encoding_matcher: re.Pattern = re.compile(
        r"<([^>]*encoding[^>]*)>\n(.*)", re.MULTILINE | re.DOTALL
    )
    parser: Literal["defusedxml", "xml"] = "defusedxml"
    """Parser to use for XML parsing. Can be either 'defusedxml' or 'xml'.
    
    * 'defusedxml' is the default parser and is used to prevent XML vulnerabilities 
       present in some distributions of Python's standard library xml.
       `defusedxml` is a wrapper around the standard library parser that
       sets up the parser with secure defaults.
    * 'xml' is the standard library parser.
    
    Use `xml` only if you are sure that your distribution of the standard library
    is not vulnerable to XML vulnerabilities. 
    
    Please review the following resources for more information:
    
    * https://docs.python.org/3/library/xml.html#xml-vulnerabilities
    * https://github.com/tiran/defusedxml 
    
    The standard library relies on libexpat for parsing XML:
    https://github.com/libexpat/libexpat 
    """
[docs]    def get_format_instructions(self) -> str:
        return XML_FORMAT_INSTRUCTIONS.format(tags=self.tags)
[docs]    def parse(self, text: str) -> Dict[str, Union[str, List[Any]]]:
        # Try to find XML string within triple backticks
        # Imports are temporarily placed here to avoid issue with caching on CI
        # likely if you're reading this you can move them to the top of the file
        if self.parser == "defusedxml":
            try:
                from defusedxml import ElementTree as DET  # type: ignore
            except ImportError:
                raise ImportError(
except ImportError:
                raise ImportError(
                    "defusedxml is not installed. "
                    "Please install it to use the defusedxml parser."
                    "You can install it with `pip install defusedxml`"
                    "See https://github.com/tiran/defusedxml for more details"
                )
            _ET = DET  # Use the defusedxml parser
        else:
            _ET = ET  # Use the standard library parser
        match = re.search(r"```(xml)?(.*)```", text, re.DOTALL)
        if match is not None:
            # If match found, use the content within the backticks
            text = match.group(2)
        encoding_match = self.encoding_matcher.search(text)
        if encoding_match:
            text = encoding_match.group(2)
        text = text.strip()
        try:
            root = ET.fromstring(text)
            return self._root_to_dict(root)
        except ET.ParseError as e:
            msg = f"Failed to parse XML format from completion {text}. Got: {e}"
            raise OutputParserException(msg, llm_output=text) from e
    def _transform(
        self, input: Iterator[Union[str, BaseMessage]]
    ) -> Iterator[AddableDict]:
        streaming_parser = _StreamingParser(self.parser)
        for chunk in input:
            yield from streaming_parser.parse(chunk)
        streaming_parser.close()
    async def _atransform(
        self, input: AsyncIterator[Union[str, BaseMessage]]
    ) -> AsyncIterator[AddableDict]:
        streaming_parser = _StreamingParser(self.parser)
        async for chunk in input:
            for output in streaming_parser.parse(chunk):
                yield output
        streaming_parser.close()
for output in streaming_parser.parse(chunk):
                yield output
        streaming_parser.close()
    def _root_to_dict(self, root: ET.Element) -> Dict[str, Union[str, List[Any]]]:
        """Converts xml tree to python dictionary."""
        if root.text and bool(re.search(r"\S", root.text)):
            # If root text contains any non-whitespace character it
            # returns {root.tag: root.text}
            return {root.tag: root.text}
        result: Dict = {root.tag: []}
        for child in root:
            if len(child) == 0:
                result[root.tag].append({child.tag: child.text})
            else:
                result[root.tag].append(self._root_to_dict(child))
        return result
    @property
    def _type(self) -> str:
        return "xml"
[docs]def nested_element(path: List[str], elem: ET.Element) -> Any:
    """Get nested element from path."""
    if len(path) == 0:
        return AddableDict({elem.tag: elem.text})
    else:
        return AddableDict({path[0]: [nested_element(path[1:], elem)]})
Source code for langchain_core.output_parsers.list
from __future__ import annotations
import re
from abc import abstractmethod
from collections import deque
from typing import AsyncIterator, Deque, Iterator, List, TypeVar, Union
from langchain_core.messages import BaseMessage
from langchain_core.output_parsers.transform import BaseTransformOutputParser
T = TypeVar("T")
[docs]def droplastn(iter: Iterator[T], n: int) -> Iterator[T]:
    """Drop the last n elements of an iterator."""
    buffer: Deque[T] = deque()
    for item in iter:
        buffer.append(item)
        if len(buffer) > n:
            yield buffer.popleft()
[docs]class ListOutputParser(BaseTransformOutputParser[List[str]]):
    """Parse the output of an LLM call to a list."""
    @property
    def _type(self) -> str:
        return "list"
[docs]    @abstractmethod
    def parse(self, text: str) -> List[str]:
        """Parse the output of an LLM call."""
[docs]    def parse_iter(self, text: str) -> Iterator[re.Match]:
        """Parse the output of an LLM call."""
        raise NotImplementedError
    def _transform(
        self, input: Iterator[Union[str, BaseMessage]]
    ) -> Iterator[List[str]]:
        buffer = ""
        for chunk in input:
            if isinstance(chunk, BaseMessage):
                # extract text
                chunk_content = chunk.content
                if not isinstance(chunk_content, str):
                    continue
                chunk = chunk_content
            # add current chunk to buffer
            buffer += chunk
            # parse buffer into a list of parts
            try:
                done_idx = 0
                # yield only complete parts
try:
                done_idx = 0
                # yield only complete parts
                for m in droplastn(self.parse_iter(buffer), 1):
                    done_idx = m.end()
                    yield [m.group(1)]
                buffer = buffer[done_idx:]
            except NotImplementedError:
                parts = self.parse(buffer)
                # yield only complete parts
                if len(parts) > 1:
                    for part in parts[:-1]:
                        yield [part]
                    buffer = parts[-1]
        # yield the last part
        for part in self.parse(buffer):
            yield [part]
    async def _atransform(
        self, input: AsyncIterator[Union[str, BaseMessage]]
    ) -> AsyncIterator[List[str]]:
        buffer = ""
        async for chunk in input:
            if isinstance(chunk, BaseMessage):
                # extract text
                chunk_content = chunk.content
                if not isinstance(chunk_content, str):
                    continue
                chunk = chunk_content
            # add current chunk to buffer
            buffer += chunk
            # parse buffer into a list of parts
            try:
                done_idx = 0
                # yield only complete parts
                for m in droplastn(self.parse_iter(buffer), 1):
                    done_idx = m.end()
                    yield [m.group(1)]
                buffer = buffer[done_idx:]
            except NotImplementedError:
                parts = self.parse(buffer)
                # yield only complete parts
                if len(parts) > 1:
                    for part in parts[:-1]:
                        yield [part]
                    buffer = parts[-1]
        # yield the last part
        for part in self.parse(buffer):
            yield [part]
[docs]class CommaSeparatedListOutputParser(ListOutputParser):
[docs]class CommaSeparatedListOutputParser(ListOutputParser):
    """Parse the output of an LLM call to a comma-separated list."""
[docs]    @classmethod
    def is_lc_serializable(cls) -> bool:
        return True
[docs]    @classmethod
    def get_lc_namespace(cls) -> List[str]:
        """Get the namespace of the langchain object."""
        return ["langchain", "output_parsers", "list"]
[docs]    def get_format_instructions(self) -> str:
        return (
            "Your response should be a list of comma separated values, "
            "eg: `foo, bar, baz` or `foo,bar,baz`"
        )
[docs]    def parse(self, text: str) -> List[str]:
        """Parse the output of an LLM call."""
        return [part.strip() for part in text.split(",")]
    @property
    def _type(self) -> str:
        return "comma-separated-list"
[docs]class NumberedListOutputParser(ListOutputParser):
    """Parse a numbered list."""
    pattern = r"\d+\.\s([^\n]+)"
[docs]    def get_format_instructions(self) -> str:
        return (
            "Your response should be a numbered list with each item on a new line. "
            "For example: \n\n1. foo\n\n2. bar\n\n3. baz"
        )
[docs]    def parse(self, text: str) -> List[str]:
        """Parse the output of an LLM call."""
        return re.findall(self.pattern, text)
[docs]    def parse_iter(self, text: str) -> Iterator[re.Match]:
        """Parse the output of an LLM call."""
"""Parse the output of an LLM call."""
        return re.finditer(self.pattern, text)
    @property
    def _type(self) -> str:
        return "numbered-list"
[docs]class MarkdownListOutputParser(ListOutputParser):
    """Parse a markdown list."""
    pattern = r"^\s*[-*]\s([^\n]+)$"
[docs]    def get_format_instructions(self) -> str:
        return "Your response should be a markdown list, " "eg: `- foo\n- bar\n- baz`"
[docs]    def parse(self, text: str) -> List[str]:
        """Parse the output of an LLM call."""
        return re.findall(self.pattern, text, re.MULTILINE)
[docs]    def parse_iter(self, text: str) -> Iterator[re.Match]:
        """Parse the output of an LLM call."""
        return re.finditer(self.pattern, text, re.MULTILINE)
    @property
    def _type(self) -> str:
        return "markdown-list"
Source code for langchain_core.output_parsers.pydantic
import json
from typing import Generic, List, Type, TypeVar, Union
import pydantic  # pydantic: ignore
from langchain_core.exceptions import OutputParserException
from langchain_core.output_parsers import JsonOutputParser
from langchain_core.outputs import Generation
from langchain_core.utils.pydantic import PYDANTIC_MAJOR_VERSION
if PYDANTIC_MAJOR_VERSION < 2:
    PydanticBaseModel = pydantic.BaseModel
else:
    from pydantic.v1 import BaseModel  # pydantic: ignore
    # Union type needs to be last assignment to PydanticBaseModel to make mypy happy.
    PydanticBaseModel = Union[BaseModel, pydantic.BaseModel]  # type: ignore
TBaseModel = TypeVar("TBaseModel", bound=PydanticBaseModel)
[docs]class PydanticOutputParser(JsonOutputParser, Generic[TBaseModel]):
    """Parse an output using a pydantic model."""
    pydantic_object: Type[TBaseModel]  # type: ignore
    """The pydantic model to parse."""
    def _parse_obj(self, obj: dict) -> TBaseModel:
        if PYDANTIC_MAJOR_VERSION == 2:
            try:
                if issubclass(self.pydantic_object, pydantic.BaseModel):
                    return self.pydantic_object.model_validate(obj)
                elif issubclass(self.pydantic_object, pydantic.v1.BaseModel):
                    return self.pydantic_object.parse_obj(obj)
                else:
                    raise OutputParserException(
                        f"Unsupported model version for PydanticOutputParser: \
                            {self.pydantic_object.__class__}"
                    )
{self.pydantic_object.__class__}"
                    )
            except (pydantic.ValidationError, pydantic.v1.ValidationError) as e:
                raise self._parser_exception(e, obj)
        else:  # pydantic v1
            try:
                return self.pydantic_object.parse_obj(obj)
            except pydantic.ValidationError as e:
                raise self._parser_exception(e, obj)
    def _parser_exception(
        self, e: Exception, json_object: dict
    ) -> OutputParserException:
        json_string = json.dumps(json_object)
        name = self.pydantic_object.__name__
        msg = f"Failed to parse {name} from completion {json_string}. Got: {e}"
        return OutputParserException(msg, llm_output=json_string)
[docs]    def parse_result(
        self, result: List[Generation], *, partial: bool = False
    ) -> TBaseModel:
        json_object = super().parse_result(result)
        return self._parse_obj(json_object)
[docs]    def parse(self, text: str) -> TBaseModel:
        return super().parse(text)
[docs]    def get_format_instructions(self) -> str:
        # Copy schema to avoid altering original Pydantic schema.
        schema = {k: v for k, v in self.pydantic_object.schema().items()}
        # Remove extraneous fields.
        reduced_schema = schema
        if "title" in reduced_schema:
            del reduced_schema["title"]
        if "type" in reduced_schema:
            del reduced_schema["type"]
        # Ensure json in context is well-formed with double quotes.
        schema_str = json.dumps(reduced_schema)
schema_str = json.dumps(reduced_schema)
        return _PYDANTIC_FORMAT_INSTRUCTIONS.format(schema=schema_str)
    @property
    def _type(self) -> str:
        return "pydantic"
    @property
    def OutputType(self) -> Type[TBaseModel]:
        """Return the pydantic model."""
        return self.pydantic_object
_PYDANTIC_FORMAT_INSTRUCTIONS = """The output should be formatted as a JSON instance that conforms to the JSON schema below.
As an example, for the schema {{"properties": {{"foo": {{"title": "Foo", "description": "a list of strings", "type": "array", "items": {{"type": "string"}}}}}}, "required": ["foo"]}}
the object {{"foo": ["bar", "baz"]}} is a well-formatted instance of the schema. The object {{"properties": {{"foo": ["bar", "baz"]}}}} is not well-formatted.
Here is the output schema:
```
{schema}
```"""  # noqa: E501
Source code for langchain_core.messages.tool
import json
from typing import Any, Dict, List, Literal, Optional, Tuple
from typing_extensions import TypedDict
from langchain_core.messages.base import (
    BaseMessage,
    BaseMessageChunk,
    merge_content,
)
from langchain_core.utils._merge import merge_dicts
[docs]class ToolMessage(BaseMessage):
    """Message for passing the result of executing a tool back to a model."""
    tool_call_id: str
    """Tool call that this message is responding to."""
    # TODO: Add is_error param?
    # is_error: bool = False
    # """Whether the tool errored."""
    type: Literal["tool"] = "tool"
[docs]    @classmethod
    def get_lc_namespace(cls) -> List[str]:
        """Get the namespace of the langchain object."""
        return ["langchain", "schema", "messages"]
ToolMessage.update_forward_refs()
[docs]class ToolMessageChunk(ToolMessage, BaseMessageChunk):
    """Tool Message chunk."""
    # Ignoring mypy re-assignment here since we're overriding the value
    # to make sure that the chunk variant can be discriminated from the
    # non-chunk variant.
    type: Literal["ToolMessageChunk"] = "ToolMessageChunk"  # type: ignore[assignment]
[docs]    @classmethod
    def get_lc_namespace(cls) -> List[str]:
        """Get the namespace of the langchain object."""
        return ["langchain", "schema", "messages"]
    def __add__(self, other: Any) -> BaseMessageChunk:  # type: ignore
        if isinstance(other, ToolMessageChunk):
            if self.tool_call_id != other.tool_call_id:
                raise ValueError(
if self.tool_call_id != other.tool_call_id:
                raise ValueError(
                    "Cannot concatenate ToolMessageChunks with different names."
                )
            return self.__class__(
                tool_call_id=self.tool_call_id,
                content=merge_content(self.content, other.content),
                additional_kwargs=merge_dicts(
                    self.additional_kwargs, other.additional_kwargs
                ),
                response_metadata=merge_dicts(
                    self.response_metadata, other.response_metadata
                ),
                id=self.id,
            )
        return super().__add__(other)
[docs]class ToolCall(TypedDict):
    """Represents a request to call a tool.
    Attributes:
        name: (str) the name of the tool to be called
        args: (dict) the arguments to the tool call
        id: (str) if provided, an identifier associated with the tool call
    """
    name: str
    args: Dict[str, Any]
    id: Optional[str]
[docs]class ToolCallChunk(TypedDict):
    """A chunk of a tool call (e.g., as part of a stream).
    When merging ToolCallChunks (e.g., via AIMessageChunk.__add__),
    all string attributes are concatenated. Chunks are only merged if their
    values of `index` are equal and not None.
    Example:
    .. code-block:: python
        left_chunks = [ToolCallChunk(name="foo", args='{"a":', index=0)]
        right_chunks = [ToolCallChunk(name=None, args='1}', index=0)]
        (
            AIMessageChunk(content="", tool_call_chunks=left_chunks)
            + AIMessageChunk(content="", tool_call_chunks=right_chunks)
+ AIMessageChunk(content="", tool_call_chunks=right_chunks)
        ).tool_call_chunks == [ToolCallChunk(name='foo', args='{"a":1}', index=0)]
    Attributes:
        name: (str) if provided, a substring of the name of the tool to be called
        args: (str) if provided, a JSON substring of the arguments to the tool call
        id: (str) if provided, a substring of an identifier for the tool call
        index: (int) if provided, the index of the tool call in a sequence
    """
    name: Optional[str]
    args: Optional[str]
    id: Optional[str]
    index: Optional[int]
[docs]class InvalidToolCall(TypedDict):
    """Allowance for errors made by LLM.
    Here we add an `error` key to surface errors made during generation
    (e.g., invalid JSON arguments.)
    """
    name: Optional[str]
    args: Optional[str]
    id: Optional[str]
    error: Optional[str]
[docs]def default_tool_parser(
    raw_tool_calls: List[dict],
) -> Tuple[List[ToolCall], List[InvalidToolCall]]:
    """Best-effort parsing of tools."""
    tool_calls = []
    invalid_tool_calls = []
    for tool_call in raw_tool_calls:
        if "function" not in tool_call:
            continue
        else:
            function_name = tool_call["function"]["name"]
            try:
                function_args = json.loads(tool_call["function"]["arguments"])
                parsed = ToolCall(
                    name=function_name or "",
                    args=function_args or {},
                    id=tool_call.get("id"),
                )
                tool_calls.append(parsed)
id=tool_call.get("id"),
                )
                tool_calls.append(parsed)
            except json.JSONDecodeError:
                invalid_tool_calls.append(
                    InvalidToolCall(
                        name=function_name,
                        args=tool_call["function"]["arguments"],
                        id=tool_call.get("id"),
                        error=None,
                    )
                )
    return tool_calls, invalid_tool_calls
[docs]def default_tool_chunk_parser(raw_tool_calls: List[dict]) -> List[ToolCallChunk]:
    """Best-effort parsing of tool chunks."""
    tool_call_chunks = []
    for tool_call in raw_tool_calls:
        if "function" not in tool_call:
            function_args = None
            function_name = None
        else:
            function_args = tool_call["function"]["arguments"]
            function_name = tool_call["function"]["name"]
        parsed = ToolCallChunk(
            name=function_name,
            args=function_args,
            id=tool_call.get("id"),
            index=tool_call.get("index"),
        )
        tool_call_chunks.append(parsed)
    return tool_call_chunks