Tony20100/langchain_full huggingface dataset:
not a Pydantic class, then the output parser will automatically extract
            only the function arguments and not the function name.
    """
    function_names = [convert_to_ernie_function(f)["name"] for f in functions]
    if isinstance(functions[0], type) and issubclass(functions[0], BaseModel):
        if len(functions) > 1:
            pydantic_schema: Union[Dict, Type[BaseModel]] = {
                name: fn for name, fn in zip(function_names, functions)
            }
        else:
            pydantic_schema = functions[0]
        output_parser: Union[
            BaseOutputParser, BaseGenerationOutputParser
        ] = PydanticOutputFunctionsParser(pydantic_schema=pydantic_schema)
    else:
        output_parser = JsonOutputFunctionsParser(args_only=len(functions) <= 1)
    return output_parser
[docs]def create_ernie_fn_runnable(
    functions: Sequence[Union[Dict[str, Any], Type[BaseModel], Callable]],
    llm: Runnable,
    prompt: BasePromptTemplate,
    *,
    output_parser: Optional[Union[BaseOutputParser, BaseGenerationOutputParser]] = None,
    **kwargs: Any,
) -> Runnable:
    """Create a runnable sequence that uses Ernie functions.
    Args:
        functions: A sequence of either dictionaries, pydantic.BaseModels classes, or
            Python functions. If dictionaries are passed in, they are assumed to
            already be a valid Ernie functions. If only a single
            function is passed in, then it will be enforced that the model use that
            function. pydantic.BaseModels and Python functions should have docstrings
            describing what the function does. For best results, pydantic.BaseModels
describing what the function does. For best results, pydantic.BaseModels
            should have descriptions of the parameters and Python functions should have
            Google Python style args descriptions in the docstring. Additionally,
            Python functions should only use primitive types (str, int, float, bool) or
            pydantic.BaseModels for arguments.
        llm: Language model to use, assumed to support the Ernie function-calling API.
        prompt: BasePromptTemplate to pass to the model.
        output_parser: BaseLLMOutputParser to use for parsing model outputs. By default
            will be inferred from the function types. If pydantic.BaseModels are passed
            in, then the OutputParser will try to parse outputs using those. Otherwise
            model outputs will simply be parsed as JSON. If multiple functions are
            passed in and they are not pydantic.BaseModels, the chain output will
            include both the name of the function that was returned and the arguments
            to pass to the function.
    Returns:
        A runnable sequence that will pass in the given functions to the model when run.
    Example:
        .. code-block:: python
                from typing import Optional
                from langchain.chains.ernie_functions import create_ernie_fn_chain
                from langchain_community.chat_models import ErnieBotChat
                from langchain_core.prompts import ChatPromptTemplate
                from langchain.pydantic_v1 import BaseModel, Field
                class RecordPerson(BaseModel):
                    \"\"\"Record some identifying information about a person.\"\"\"
                    name: str = Field(..., description="The person's name")
                    age: int = Field(..., description="The person's age")
                    fav_food: Optional[str] = Field(None, description="The person's favorite food")
                class RecordDog(BaseModel):
class RecordDog(BaseModel):
                    \"\"\"Record some identifying information about a dog.\"\"\"
                    name: str = Field(..., description="The dog's name")
                    color: str = Field(..., description="The dog's color")
                    fav_food: Optional[str] = Field(None, description="The dog's favorite food")
                llm = ErnieBotChat(model_name="ERNIE-Bot-4")
                prompt = ChatPromptTemplate.from_messages(
                    [
                        ("user", "Make calls to the relevant function to record the entities in the following input: {input}"),
                        ("assistant", "OK!"),
                        ("user", "Tip: Make sure to answer in the correct format"),
                    ]
                )
                chain = create_ernie_fn_runnable([RecordPerson, RecordDog], llm, prompt)
                chain.invoke({"input": "Harry was a chubby brown beagle who loved chicken"})
                # -> RecordDog(name="Harry", color="brown", fav_food="chicken")
    """  # noqa: E501
    if not functions:
        raise ValueError("Need to pass in at least one function. Received zero.")
    ernie_functions = [convert_to_ernie_function(f) for f in functions]
    llm_kwargs: Dict[str, Any] = {"functions": ernie_functions, **kwargs}
    if len(ernie_functions) == 1:
        llm_kwargs["function_call"] = {"name": ernie_functions[0]["name"]}
    output_parser = output_parser or get_ernie_output_parser(functions)
    return prompt | llm.bind(**llm_kwargs) | output_parser
[docs]def create_structured_output_runnable(
    output_schema: Union[Dict[str, Any], Type[BaseModel]],
output_schema: Union[Dict[str, Any], Type[BaseModel]],
    llm: Runnable,
    prompt: BasePromptTemplate,
    *,
    output_parser: Optional[Union[BaseOutputParser, BaseGenerationOutputParser]] = None,
    **kwargs: Any,
) -> Runnable:
    """Create a runnable that uses an Ernie function to get a structured output.
    Args:
        output_schema: Either a dictionary or pydantic.BaseModel class. If a dictionary
            is passed in, it's assumed to already be a valid JsonSchema.
            For best results, pydantic.BaseModels should have docstrings describing what
            the schema represents and descriptions for the parameters.
        llm: Language model to use, assumed to support the Ernie function-calling API.
        prompt: BasePromptTemplate to pass to the model.
        output_parser: BaseLLMOutputParser to use for parsing model outputs. By default
            will be inferred from the function types. If pydantic.BaseModels are passed
            in, then the OutputParser will try to parse outputs using those. Otherwise
            model outputs will simply be parsed as JSON.
    Returns:
        A runnable sequence that will pass the given function to the model when run.
    Example:
        .. code-block:: python
            from typing import Optional
            from langchain.chains.ernie_functions import create_structured_output_chain
            from langchain_community.chat_models import ErnieBotChat
            from langchain_core.prompts import ChatPromptTemplate
            from langchain.pydantic_v1 import BaseModel, Field
            class Dog(BaseModel):
                \"\"\"Identifying information about a dog.\"\"\"
                name: str = Field(..., description="The dog's name")
                color: str = Field(..., description="The dog's color")
color: str = Field(..., description="The dog's color")
                fav_food: Optional[str] = Field(None, description="The dog's favorite food")
            llm = ErnieBotChat(model_name="ERNIE-Bot-4")
            prompt = ChatPromptTemplate.from_messages(
                [
                    ("user", "Use the given format to extract information from the following input: {input}"),
                    ("assistant", "OK!"),
                    ("user", "Tip: Make sure to answer in the correct format"),
                ]
            )
            chain = create_structured_output_chain(Dog, llm, prompt)
            chain.invoke({"input": "Harry was a chubby brown beagle who loved chicken"})
            # -> Dog(name="Harry", color="brown", fav_food="chicken")
    """  # noqa: E501
    if isinstance(output_schema, dict):
        function: Any = {
            "name": "output_formatter",
            "description": (
                "Output formatter. Should always be used to format your response to the"
                " user."
            ),
            "parameters": output_schema,
        }
    else:
        class _OutputFormatter(BaseModel):
            """Output formatter. Should always be used to format your response to the user."""  # noqa: E501
            output: output_schema  # type: ignore
        function = _OutputFormatter
        output_parser = output_parser or PydanticAttrOutputFunctionsParser(
            pydantic_schema=_OutputFormatter, attr_name="output"
        )
    return create_ernie_fn_runnable(
        [function],
        llm,
        prompt,
        output_parser=output_parser,
        **kwargs,
    )
""" --- Legacy --- """
[docs]def create_ernie_fn_chain(
""" --- Legacy --- """
[docs]def create_ernie_fn_chain(
    functions: Sequence[Union[Dict[str, Any], Type[BaseModel], Callable]],
    llm: BaseLanguageModel,
    prompt: BasePromptTemplate,
    *,
    output_key: str = "function",
    output_parser: Optional[BaseLLMOutputParser] = None,
    **kwargs: Any,
) -> LLMChain:  # type: ignore[valid-type]
    """[Legacy] Create an LLM chain that uses Ernie functions.
    Args:
        functions: A sequence of either dictionaries, pydantic.BaseModels classes, or
            Python functions. If dictionaries are passed in, they are assumed to
            already be a valid Ernie functions. If only a single
            function is passed in, then it will be enforced that the model use that
            function. pydantic.BaseModels and Python functions should have docstrings
            describing what the function does. For best results, pydantic.BaseModels
            should have descriptions of the parameters and Python functions should have
            Google Python style args descriptions in the docstring. Additionally,
            Python functions should only use primitive types (str, int, float, bool) or
            pydantic.BaseModels for arguments.
        llm: Language model to use, assumed to support the Ernie function-calling API.
        prompt: BasePromptTemplate to pass to the model.
        output_key: The key to use when returning the output in LLMChain.__call__.
        output_parser: BaseLLMOutputParser to use for parsing model outputs. By default
            will be inferred from the function types. If pydantic.BaseModels are passed
            in, then the OutputParser will try to parse outputs using those. Otherwise
in, then the OutputParser will try to parse outputs using those. Otherwise
            model outputs will simply be parsed as JSON. If multiple functions are
            passed in and they are not pydantic.BaseModels, the chain output will
            include both the name of the function that was returned and the arguments
            to pass to the function.
    Returns:
        An LLMChain that will pass in the given functions to the model when run.
    Example:
        .. code-block:: python
                from typing import Optional
                from langchain.chains.ernie_functions import create_ernie_fn_chain
                from langchain_community.chat_models import ErnieBotChat
                from langchain_core.prompts import ChatPromptTemplate
                from langchain.pydantic_v1 import BaseModel, Field
                class RecordPerson(BaseModel):
                    \"\"\"Record some identifying information about a person.\"\"\"
                    name: str = Field(..., description="The person's name")
                    age: int = Field(..., description="The person's age")
                    fav_food: Optional[str] = Field(None, description="The person's favorite food")
                class RecordDog(BaseModel):
                    \"\"\"Record some identifying information about a dog.\"\"\"
                    name: str = Field(..., description="The dog's name")
                    color: str = Field(..., description="The dog's color")
                    fav_food: Optional[str] = Field(None, description="The dog's favorite food")
                llm = ErnieBotChat(model_name="ERNIE-Bot-4")
                prompt = ChatPromptTemplate.from_messages(
                    [
                        ("user", "Make calls to the relevant function to record the entities in the following input: {input}"),
                        ("assistant", "OK!"),
("assistant", "OK!"),
                        ("user", "Tip: Make sure to answer in the correct format"),
                    ]
                )
                chain = create_ernie_fn_chain([RecordPerson, RecordDog], llm, prompt)
                chain.run("Harry was a chubby brown beagle who loved chicken")
                # -> RecordDog(name="Harry", color="brown", fav_food="chicken")
    """  # noqa: E501
    if not functions:
        raise ValueError("Need to pass in at least one function. Received zero.")
    ernie_functions = [convert_to_ernie_function(f) for f in functions]
    output_parser = output_parser or get_ernie_output_parser(functions)
    llm_kwargs: Dict[str, Any] = {
        "functions": ernie_functions,
    }
    if len(ernie_functions) == 1:
        llm_kwargs["function_call"] = {"name": ernie_functions[0]["name"]}
    llm_chain = LLMChain(  # type: ignore[misc]
        llm=llm,
        prompt=prompt,
        output_parser=output_parser,
        llm_kwargs=llm_kwargs,
        output_key=output_key,
        **kwargs,
    )
    return llm_chain
[docs]def create_structured_output_chain(
    output_schema: Union[Dict[str, Any], Type[BaseModel]],
    llm: BaseLanguageModel,
    prompt: BasePromptTemplate,
    *,
    output_key: str = "function",
    output_parser: Optional[BaseLLMOutputParser] = None,
    **kwargs: Any,
) -> LLMChain:  # type: ignore[valid-type]
) -> LLMChain:  # type: ignore[valid-type]
    """[Legacy] Create an LLMChain that uses an Ernie function to get a structured output.
    Args:
        output_schema: Either a dictionary or pydantic.BaseModel class. If a dictionary
            is passed in, it's assumed to already be a valid JsonSchema.
            For best results, pydantic.BaseModels should have docstrings describing what
            the schema represents and descriptions for the parameters.
        llm: Language model to use, assumed to support the Ernie function-calling API.
        prompt: BasePromptTemplate to pass to the model.
        output_key: The key to use when returning the output in LLMChain.__call__.
        output_parser: BaseLLMOutputParser to use for parsing model outputs. By default
            will be inferred from the function types. If pydantic.BaseModels are passed
            in, then the OutputParser will try to parse outputs using those. Otherwise
            model outputs will simply be parsed as JSON.
    Returns:
        An LLMChain that will pass the given function to the model.
    Example:
        .. code-block:: python
                from typing import Optional
                from langchain.chains.ernie_functions import create_structured_output_chain
                from langchain_community.chat_models import ErnieBotChat
                from langchain_core.prompts import ChatPromptTemplate
                from langchain.pydantic_v1 import BaseModel, Field
                class Dog(BaseModel):
                    \"\"\"Identifying information about a dog.\"\"\"
                    name: str = Field(..., description="The dog's name")
                    color: str = Field(..., description="The dog's color")
                    fav_food: Optional[str] = Field(None, description="The dog's favorite food")
fav_food: Optional[str] = Field(None, description="The dog's favorite food")
                llm = ErnieBotChat(model_name="ERNIE-Bot-4")
                prompt = ChatPromptTemplate.from_messages(
                    [
                        ("user", "Use the given format to extract information from the following input: {input}"),
                        ("assistant", "OK!"),
                        ("user", "Tip: Make sure to answer in the correct format"),
                    ]
                )
                chain = create_structured_output_chain(Dog, llm, prompt)
                chain.run("Harry was a chubby brown beagle who loved chicken")
                # -> Dog(name="Harry", color="brown", fav_food="chicken")
    """  # noqa: E501
    if isinstance(output_schema, dict):
        function: Any = {
            "name": "output_formatter",
            "description": (
                "Output formatter. Should always be used to format your response to the"
                " user."
            ),
            "parameters": output_schema,
        }
    else:
        class _OutputFormatter(BaseModel):
            """Output formatter. Should always be used to format your response to the user."""  # noqa: E501
            output: output_schema  # type: ignore
        function = _OutputFormatter
        output_parser = output_parser or PydanticAttrOutputFunctionsParser(
            pydantic_schema=_OutputFormatter, attr_name="output"
        )
    return create_ernie_fn_chain(
        [function],
        llm,
        prompt,
        output_key=output_key,
        output_parser=output_parser,
        **kwargs,
    )
Source code for langchain_community.chains.pebblo_retrieval.models
"""Models for the PebbloRetrievalQA chain."""
from typing import Any, List, Optional
from langchain_core.pydantic_v1 import BaseModel
[docs]class AuthContext(BaseModel):
    """Class for an authorization context."""
    name: Optional[str] = None
    user_id: str
    user_auth: List[str]
    """List of user authorizations, which may include their User ID and 
    the groups they are part of"""
[docs]class SemanticEntities(BaseModel):
    """Class for a semantic entity filter."""
    deny: List[str]
[docs]class SemanticTopics(BaseModel):
    """Class for a semantic topic filter."""
    deny: List[str]
[docs]class SemanticContext(BaseModel):
    """Class for a semantic context."""
    pebblo_semantic_entities: Optional[SemanticEntities] = None
    pebblo_semantic_topics: Optional[SemanticTopics] = None
    def __init__(self, **data: Any) -> None:
        super().__init__(**data)
        # Validate semantic_context
        if (
            self.pebblo_semantic_entities is None
            and self.pebblo_semantic_topics is None
        ):
            raise ValueError(
                "semantic_context must contain 'pebblo_semantic_entities' or "
                "'pebblo_semantic_topics'"
            )
[docs]class ChainInput(BaseModel):
    """Input for PebbloRetrievalQA chain."""
    query: str
    auth_context: Optional[AuthContext] = None
    semantic_context: Optional[SemanticContext] = None
[docs]    def dict(self, **kwargs: Any) -> dict:
        base_dict = super().dict(**kwargs)
base_dict = super().dict(**kwargs)
        # Keep auth_context and semantic_context as it is(Pydantic models)
        base_dict["auth_context"] = self.auth_context
        base_dict["semantic_context"] = self.semantic_context
        return base_dict
[docs]class Runtime(BaseModel):
    """
    OS, language details
    """
    type: Optional[str] = ""
    host: str
    path: str
    ip: Optional[str] = ""
    platform: str
    os: str
    os_version: str
    language: str
    language_version: str
    runtime: Optional[str] = ""
[docs]class Framework(BaseModel):
    """
    Langchain framework details
    """
    name: str
    version: str
[docs]class Model(BaseModel):
    vendor: Optional[str]
    name: Optional[str]
[docs]class PkgInfo(BaseModel):
    project_home_page: Optional[str]
    documentation_url: Optional[str]
    pypi_url: Optional[str]
    liscence_type: Optional[str]
    installed_via: Optional[str]
    location: Optional[str]
[docs]class VectorDB(BaseModel):
    name: Optional[str] = None
    version: Optional[str] = None
    location: Optional[str] = None
    embedding_model: Optional[str] = None
[docs]class Chains(BaseModel):
    name: str
    model: Optional[Model]
    vector_dbs: Optional[List[VectorDB]]
[docs]class App(BaseModel):
    name: str
    owner: str
    description: Optional[str]
    runtime: Runtime
    framework: Framework
    chains: List[Chains]
    plugin_version: str
[docs]class Context(BaseModel):
plugin_version: str
[docs]class Context(BaseModel):
    retrieved_from: Optional[str]
    doc: Optional[str]
    vector_db: str
[docs]class Prompt(BaseModel):
    data: str
[docs]class Qa(BaseModel):
    name: str
    context: List[Optional[Context]]
    prompt: Prompt
    response: Prompt
    prompt_time: str
    user: str
    user_identities: Optional[List[str]]
Source code for langchain_community.chains.pebblo_retrieval.base
"""
Pebblo Retrieval Chain with Identity & Semantic Enforcement for question-answering
against a vector database.
"""
import datetime
import inspect
import logging
from http import HTTPStatus
from typing import Any, Dict, List, Optional
import requests  # type: ignore
from langchain.chains.base import Chain
from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
from langchain_core.callbacks import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
)
from langchain_core.documents import Document
from langchain_core.language_models import BaseLanguageModel
from langchain_core.pydantic_v1 import Extra, Field, validator
from langchain_core.vectorstores import VectorStoreRetriever
from langchain_community.chains.pebblo_retrieval.enforcement_filters import (
    SUPPORTED_VECTORSTORES,
    set_enforcement_filters,
)
from langchain_community.chains.pebblo_retrieval.models import (
    App,
    AuthContext,
    Qa,
    SemanticContext,
)
from langchain_community.chains.pebblo_retrieval.utilities import (
    APP_DISCOVER_URL,
    CLASSIFIER_URL,
    PEBBLO_CLOUD_URL,
    PLUGIN_VERSION,
    PROMPT_URL,
    get_runtime,
)
logger = logging.getLogger(__name__)
[docs]class PebbloRetrievalQA(Chain):
    """
    Retrieval Chain with Identity & Semantic Enforcement for question-answering
    against a vector database.
    """
    combine_documents_chain: BaseCombineDocumentsChain
    """Chain to use to combine the documents."""
    input_key: str = "query"  #: :meta private:
input_key: str = "query"  #: :meta private:
    output_key: str = "result"  #: :meta private:
    return_source_documents: bool = False
    """Return the source documents or not."""
    retriever: VectorStoreRetriever = Field(exclude=True)
    """VectorStore to use for retrieval."""
    auth_context_key: str = "auth_context"  #: :meta private:
    """Authentication context for identity enforcement."""
    semantic_context_key: str = "semantic_context"  #: :meta private:
    """Semantic context for semantic enforcement."""
    app_name: str  #: :meta private:
    """App name."""
    owner: str  #: :meta private:
    """Owner of app."""
    description: str  #: :meta private:
    """Description of app."""
    api_key: Optional[str] = None  #: :meta private:
    """Pebblo cloud API key for app."""
    classifier_url: str = CLASSIFIER_URL  #: :meta private:
    """Classifier endpoint."""
    _discover_sent = False  #: :meta private:
    """Flag to check if discover payload has been sent."""
    _prompt_sent = False  #: :meta private:
    """Flag to check if prompt payload has been sent."""
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run get_relevant_text and llm on input query.
        If chain has 'return_source_documents' as 'True', returns
        the retrieved documents as well under the key 'source_documents'.
        Example:
        .. code-block:: python