viruPVS/langchain-docs huggingface dataset:
Source code for langchain_experimental.plan_and_execute.planners.chat_planner
import re
from langchain.chains import LLMChain
from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
from langchain.schema.language_model import BaseLanguageModel
from langchain.schema.messages import SystemMessage
from langchain_experimental.plan_and_execute.planners.base import LLMPlanner
from langchain_experimental.plan_and_execute.schema import (
    Plan,
    PlanOutputParser,
    Step,
)
SYSTEM_PROMPT = (
    "Let's first understand the problem and devise a plan to solve the problem."
    " Please output the plan starting with the header 'Plan:' "
    "and then followed by a numbered list of steps. "
    "Please make the plan the minimum number of steps required "
    "to accurately complete the task. If the task is a question, "
    "the final step should almost always be 'Given the above steps taken, "
    "please respond to the users original question'. "
    "At the end of your plan, say '<END_OF_PLAN>'"
)
[docs]class PlanningOutputParser(PlanOutputParser):
    """Planning output parser."""
[docs]    def parse(self, text: str) -> Plan:
        steps = [Step(value=v) for v in re.split("\n\s*\d+\. ", text)[1:]]
        return Plan(steps=steps)
[docs]def load_chat_planner(
    llm: BaseLanguageModel, system_prompt: str = SYSTEM_PROMPT
) -> LLMPlanner:
    """
    Load a chat planner.
    Args:
        llm: Language model.
        system_prompt: System prompt.
    Returns:
        LLMPlanner
    """
Returns:
        LLMPlanner
    """
    prompt_template = ChatPromptTemplate.from_messages(
        [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template("{input}"),
        ]
    )
    llm_chain = LLMChain(llm=llm, prompt=prompt_template)
    return LLMPlanner(
        llm_chain=llm_chain,
        output_parser=PlanningOutputParser(),
        stop=["<END_OF_PLAN>"],
    )
Source code for langchain_experimental.plan_and_execute.executors.base
from abc import abstractmethod
from typing import Any
from langchain.callbacks.manager import Callbacks
from langchain.chains.base import Chain
from langchain_experimental.plan_and_execute.schema import StepResponse
from langchain_experimental.pydantic_v1 import BaseModel
[docs]class BaseExecutor(BaseModel):
    """Base executor."""
[docs]    @abstractmethod
    def step(
        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
    ) -> StepResponse:
        """Take step."""
[docs]    @abstractmethod
    async def astep(
        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
    ) -> StepResponse:
        """Take async step."""
[docs]class ChainExecutor(BaseExecutor):
    """Chain executor."""
    chain: Chain
    """The chain to use."""
[docs]    def step(
        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
    ) -> StepResponse:
        """Take step."""
        response = self.chain.run(**inputs, callbacks=callbacks)
        return StepResponse(response=response)
[docs]    async def astep(
        self, inputs: dict, callbacks: Callbacks = None, **kwargs: Any
    ) -> StepResponse:
        """Take step."""
        response = await self.chain.arun(**inputs, callbacks=callbacks)
        return StepResponse(response=response)
Source code for langchain_experimental.plan_and_execute.executors.agent_executor
from typing import List
from langchain.agents.agent import AgentExecutor
from langchain.agents.structured_chat.base import StructuredChatAgent
from langchain.schema.language_model import BaseLanguageModel
from langchain.tools import BaseTool
from langchain_experimental.plan_and_execute.executors.base import ChainExecutor
HUMAN_MESSAGE_TEMPLATE = """Previous steps: {previous_steps}
Current objective: {current_step}
{agent_scratchpad}"""
TASK_PREFIX = """{objective}
"""
[docs]def load_agent_executor(
    llm: BaseLanguageModel,
    tools: List[BaseTool],
    verbose: bool = False,
    include_task_in_prompt: bool = False,
) -> ChainExecutor:
    """
    Load an agent executor.
    Args:
        llm: BaseLanguageModel
        tools: List[BaseTool]
        verbose: bool. Defaults to False.
        include_task_in_prompt: bool. Defaults to False.
    Returns:
        ChainExecutor
    """
    input_variables = ["previous_steps", "current_step", "agent_scratchpad"]
    template = HUMAN_MESSAGE_TEMPLATE
    if include_task_in_prompt:
        input_variables.append("objective")
        template = TASK_PREFIX + template
    agent = StructuredChatAgent.from_llm_and_tools(
        llm,
        tools,
        human_message_template=template,
        input_variables=input_variables,
    )
    agent_executor = AgentExecutor.from_agent_and_tools(
        agent=agent, tools=tools, verbose=verbose
    )
    return ChainExecutor(chain=agent_executor)
Source code for langchain_experimental.tools.python.tool
"""A tool for running python code in a REPL."""
import ast
import asyncio
import re
import sys
from contextlib import redirect_stdout
from io import StringIO
from typing import Any, Dict, Optional, Type
from langchain.callbacks.manager import (
    AsyncCallbackManagerForToolRun,
    CallbackManagerForToolRun,
)
from langchain.pydantic_v1 import BaseModel, Field, root_validator
from langchain.tools.base import BaseTool
from langchain_experimental.utilities.python import PythonREPL
def _get_default_python_repl() -> PythonREPL:
    return PythonREPL(_globals=globals(), _locals=None)
[docs]def sanitize_input(query: str) -> str:
    """Sanitize input to the python REPL.
    Remove whitespace, backtick & python (if llm mistakes python console as terminal)
    Args:
        query: The query to sanitize
    Returns:
        str: The sanitized query
    """
    # Removes `, whitespace & python from start
    query = re.sub(r"^(\s|`)*(?i:python)?\s*", "", query)
    # Removes whitespace & ` from end
    query = re.sub(r"(\s|`)*$", "", query)
    return query
[docs]class PythonREPLTool(BaseTool):
    """A tool for running python code in a REPL."""
    name: str = "Python_REPL"
    description: str = (
        "A Python shell. Use this to execute python commands. "
        "Input should be a valid python command. "
        "If you want to see the output of a value, you should print it out "
        "with `print(...)`."
    )
"with `print(...)`."
    )
    python_repl: PythonREPL = Field(default_factory=_get_default_python_repl)
    sanitize_input: bool = True
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> Any:
        """Use the tool."""
        if self.sanitize_input:
            query = sanitize_input(query)
        return self.python_repl.run(query)
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> Any:
        """Use the tool asynchronously."""
        if self.sanitize_input:
            query = sanitize_input(query)
        loop = asyncio.get_running_loop()
        result = await loop.run_in_executor(None, self.run, query)
        return result
[docs]class PythonInputs(BaseModel):
    query: str = Field(description="code snippet to run")
[docs]class PythonAstREPLTool(BaseTool):
    """A tool for running python code in a REPL."""
    name: str = "python_repl_ast"
    description: str = (
        "A Python shell. Use this to execute python commands. "
        "Input should be a valid python command. "
        "When using this tool, sometimes output is abbreviated - "
        "make sure it does not look abbreviated before using it in your answer."
    )
    globals: Optional[Dict] = Field(default_factory=dict)
    locals: Optional[Dict] = Field(default_factory=dict)
    sanitize_input: bool = True
    args_schema: Type[BaseModel] = PythonInputs
    @root_validator(pre=True)
@root_validator(pre=True)
    def validate_python_version(cls, values: Dict) -> Dict:
        """Validate valid python version."""
        if sys.version_info < (3, 9):
            raise ValueError(
                "This tool relies on Python 3.9 or higher "
                "(as it uses new functionality in the `ast` module, "
                f"you have Python version: {sys.version}"
            )
        return values
    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        try:
            if self.sanitize_input:
                query = sanitize_input(query)
            tree = ast.parse(query)
            module = ast.Module(tree.body[:-1], type_ignores=[])
            exec(ast.unparse(module), self.globals, self.locals)  # type: ignore
            module_end = ast.Module(tree.body[-1:], type_ignores=[])
            module_end_str = ast.unparse(module_end)  # type: ignore
            io_buffer = StringIO()
            try:
                with redirect_stdout(io_buffer):
                    ret = eval(module_end_str, self.globals, self.locals)
                    if ret is None:
                        return io_buffer.getvalue()
                    else:
                        return ret
            except Exception:
                with redirect_stdout(io_buffer):
                    exec(module_end_str, self.globals, self.locals)
                return io_buffer.getvalue()
        except Exception as e:
            return "{}: {}".format(type(e).__name__, str(e))
    async def _arun(
        self,
        query: str,
        run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
    ) -> Any:
Source code for langchain_experimental.prompt_injection_identifier.hugging_face_identifier
"""Tool for the identification of prompt injection attacks."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from langchain.pydantic_v1 import Field, root_validator
from langchain.tools.base import BaseTool
if TYPE_CHECKING:
    from transformers import Pipeline
def _model_default_factory(
    model_name: str = "deepset/deberta-v3-base-injection"
) -> Pipeline:
    try:
        from transformers import pipeline
    except ImportError as e:
        raise ImportError(
            "Cannot import transformers, please install with "
            "`pip install transformers`."
        ) from e
    return pipeline("text-classification", model=model_name)
[docs]class HuggingFaceInjectionIdentifier(BaseTool):
    """Tool that uses HF model to detect prompt injection attacks."""
    name: str = "hugging_face_injection_identifier"
    description: str = (
        "A wrapper around HuggingFace Prompt Injection security model. "
        "Useful for when you need to ensure that prompt is free of injection attacks. "
        "Input should be any message from the user."
    )
    model: Any = Field(default_factory=_model_default_factory)
    """Model to use for prompt injection detection. 
    
    Can be specified as transformers Pipeline or string. String should correspond to the
        model name of a text-classification transformers model. Defaults to 
        ``deepset/deberta-v3-base-injection`` model.
    """
    @root_validator(pre=True)
    def validate_environment(cls, values: dict) -> dict:
        if isinstance(values.get("model"), str):
            values["model"] = _model_default_factory(model_name=values["model"])
        return values
Source code for langchain_experimental.llm_symbolic_math.base
"""Chain that interprets a prompt and executes python code to do symbolic math."""
from __future__ import annotations
import re
from typing import Any, Dict, List, Optional
from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
)
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.prompts.base import BasePromptTemplate
from langchain_experimental.llm_symbolic_math.prompt import PROMPT
from langchain_experimental.pydantic_v1 import Extra
[docs]class LLMSymbolicMathChain(Chain):
    """Chain that interprets a prompt and executes python code to do symbolic math.
    Example:
        .. code-block:: python
            from langchain.chains import LLMSymbolicMathChain
            from langchain.llms import OpenAI
            llm_symbolic_math = LLMSymbolicMathChain.from_llm(OpenAI())
    """
    llm_chain: LLMChain
    input_key: str = "question"  #: :meta private:
    output_key: str = "answer"  #: :meta private:
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @property
    def input_keys(self) -> List[str]:
        """Expect input key.
        :meta private:
        """
        return [self.input_key]
    @property
    def output_keys(self) -> List[str]:
        """Expect output key.
        :meta private:
        """
        return [self.output_key]
:meta private:
        """
        return [self.output_key]
    def _evaluate_expression(self, expression: str) -> str:
        try:
            import sympy
        except ImportError as e:
            raise ImportError(
                "Unable to import sympy, please install it with `pip install sympy`."
            ) from e
        try:
            output = str(sympy.sympify(expression, evaluate=True))
        except Exception as e:
            raise ValueError(
                f'LLMSymbolicMathChain._evaluate("{expression}") raised error: {e}.'
                " Please try again with a valid numerical expression"
            )
        # Remove any leading and trailing brackets from the output
        return re.sub(r"^\[|\]$", "", output)
    def _process_llm_result(
        self, llm_output: str, run_manager: CallbackManagerForChainRun
    ) -> Dict[str, str]:
        run_manager.on_text(llm_output, color="green", verbose=self.verbose)
        llm_output = llm_output.strip()
        text_match = re.search(r"^```text(.*?)```", llm_output, re.DOTALL)
        if text_match:
            expression = text_match.group(1)
            output = self._evaluate_expression(expression)
            run_manager.on_text("\nAnswer: ", verbose=self.verbose)
            run_manager.on_text(output, color="yellow", verbose=self.verbose)
            answer = "Answer: " + output
        elif llm_output.startswith("Answer:"):
            answer = llm_output
        elif "Answer:" in llm_output:
            answer = "Answer: " + llm_output.split("Answer:")[-1]
        else:
            raise ValueError(f"unknown format from LLM: {llm_output}")
raise ValueError(f"unknown format from LLM: {llm_output}")
        return {self.output_key: answer}
    async def _aprocess_llm_result(
        self,
        llm_output: str,
        run_manager: AsyncCallbackManagerForChainRun,
    ) -> Dict[str, str]:
        await run_manager.on_text(llm_output, color="green", verbose=self.verbose)
        llm_output = llm_output.strip()
        text_match = re.search(r"^```text(.*?)```", llm_output, re.DOTALL)
        if text_match:
            expression = text_match.group(1)
            output = self._evaluate_expression(expression)
            await run_manager.on_text("\nAnswer: ", verbose=self.verbose)
            await run_manager.on_text(output, color="yellow", verbose=self.verbose)
            answer = "Answer: " + output
        elif llm_output.startswith("Answer:"):
            answer = llm_output
        elif "Answer:" in llm_output:
            answer = "Answer: " + llm_output.split("Answer:")[-1]
        else:
            raise ValueError(f"unknown format from LLM: {llm_output}")
        return {self.output_key: answer}
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_key])
        llm_output = self.llm_chain.predict(
            question=inputs[self.input_key],
            stop=["```output"],
            callbacks=_run_manager.get_child(),
        )
stop=["```output"],
            callbacks=_run_manager.get_child(),
        )
        return self._process_llm_result(llm_output, _run_manager)
    async def _acall(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        await _run_manager.on_text(inputs[self.input_key])
        llm_output = await self.llm_chain.apredict(
            question=inputs[self.input_key],
            stop=["```output"],
            callbacks=_run_manager.get_child(),
        )
        return await self._aprocess_llm_result(llm_output, _run_manager)
    @property
    def _chain_type(self) -> str:
        return "llm_symbolic_math_chain"
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: BasePromptTemplate = PROMPT,
        **kwargs: Any,
    ) -> LLMSymbolicMathChain:
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(llm_chain=llm_chain, **kwargs)
Source code for langchain_experimental.tabular_synthetic_data.openai
from typing import Any, Dict, Optional, Type, Union
from langchain.chains.openai_functions import create_structured_output_chain
from langchain.chat_models import ChatOpenAI
from langchain.prompts import PromptTemplate
from langchain.pydantic_v1 import BaseModel
from langchain.schema import BaseLLMOutputParser, BasePromptTemplate
from langchain_experimental.tabular_synthetic_data.base import SyntheticDataGenerator
OPENAI_TEMPLATE = PromptTemplate(input_variables=["example"], template="{example}")
[docs]def create_openai_data_generator(
    output_schema: Union[Dict[str, Any], Type[BaseModel]],
    llm: ChatOpenAI,
    prompt: BasePromptTemplate,
    output_parser: Optional[BaseLLMOutputParser] = None,
    **kwargs: Any,
) -> SyntheticDataGenerator:
    """
    Create an instance of SyntheticDataGenerator tailored for OpenAI models.
    This function creates an LLM chain designed for structured output based on the
    provided schema, language model, and prompt template. The resulting chain is then
    used to instantiate and return a SyntheticDataGenerator.
    Args:
        output_schema (Union[Dict[str, Any], Type[BaseModel]]): Schema for expected
        output. This can be either a dictionary representing a valid JsonSchema or a
        Pydantic BaseModel class.
        llm (ChatOpenAI): OpenAI language model to use.
        prompt (BasePromptTemplate): Template to be used for generating prompts.
        output_parser (Optional[BaseLLMOutputParser], optional): Parser for
        processing model outputs. If none is provided, a default will be inferred
        from the function types.
        **kwargs: Additional keyword arguments to be passed to
        `create_structured_output_chain`.
`create_structured_output_chain`.
    Returns: SyntheticDataGenerator: An instance of the data generator set up with
    the constructed chain.
    Usage:
        To generate synthetic data with a structured output, first define your desired
        output schema. Then, use this function to create a SyntheticDataGenerator
        instance. After obtaining the generator, you can utilize its methods to produce
        the desired synthetic data.
    """
    # Create function calling chain to ensure structured output
    chain = create_structured_output_chain(
        output_schema, llm, prompt, output_parser=output_parser, **kwargs
    )
    # Create the SyntheticDataGenerator instance with the created chain
    generator = SyntheticDataGenerator(template=prompt, llm_chain=chain)
    return generator
Source code for langchain_experimental.tabular_synthetic_data.base
import asyncio
from typing import Any, Dict, List, Optional, Union
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.prompts.few_shot import FewShotPromptTemplate
from langchain.pydantic_v1 import BaseModel, root_validator
from langchain.schema.language_model import BaseLanguageModel
[docs]class SyntheticDataGenerator(BaseModel):
    """Generates synthetic data using the given LLM and few-shot template.
    Utilizes the provided LLM to produce synthetic data based on the
    few-shot prompt template.
    Attributes:
        template (FewShotPromptTemplate): Template for few-shot prompting.
        llm (Optional[BaseLanguageModel]): Large Language Model to use for generation.
        llm_chain (Optional[Chain]): LLM chain with the LLM and few-shot template.
        example_input_key (str): Key to use for storing example inputs.
    Usage Example:
        >>> template = FewShotPromptTemplate(...)
        >>> llm = BaseLanguageModel(...)
        >>> generator = SyntheticDataGenerator(template=template, llm=llm)
        >>> results = generator.generate(subject="climate change", runs=5)
    """
    template: FewShotPromptTemplate
    llm: Optional[BaseLanguageModel] = None
    results: list = []
    llm_chain: Optional[Chain] = None
    example_input_key: str = "example"
    class Config:
        validate_assignment = True
    @root_validator(pre=False, skip_on_failure=True)
    def set_llm_chain(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        llm_chain = values.get("llm_chain")
        llm = values.get("llm")
llm = values.get("llm")
        few_shot_template = values.get("template")
        if not llm_chain:  # If llm_chain is None or not present
            if llm is None or few_shot_template is None:
                raise ValueError(
                    "Both llm and few_shot_template must be provided if llm_chain is "
                    "not given."
                )
            values["llm_chain"] = LLMChain(llm=llm, prompt=few_shot_template)
        return values
    @staticmethod
    def _format_dict_to_string(input_dict: Dict) -> str:
        formatted_str = ", ".join(
            [f"{key}: {value}" for key, value in input_dict.items()]
        )
        return formatted_str
    def _update_examples(self, example: Union[BaseModel, Dict[str, Any], str]) -> None:
        """Prevents duplicates by adding previously generated examples to the few shot
        list."""
        if self.template and self.template.examples:
            if isinstance(example, BaseModel):
                formatted_example = self._format_dict_to_string(example.dict())
            elif isinstance(example, dict):
                formatted_example = self._format_dict_to_string(example)
            else:
                formatted_example = str(example)
            self.template.examples.pop(0)
            self.template.examples.append({self.example_input_key: formatted_example})
[docs]    def generate(self, subject: str, runs: int, *args: Any, **kwargs: Any) -> List[str]:
        """Generate synthetic data using the given subject string.
        Args:
            subject (str): The subject the synthetic data will be about.
            runs (int): Number of times to generate the data.
            extra (str): Extra instructions for steerability in data generation.
        Returns: