viruPVS/langchain-docs huggingface dataset:
self.process = None
        if persistent:
            self.prompt = str(uuid4())
            self.process = self._initialize_persistent_process(self, self.prompt)
    @staticmethod
    def _lazy_import_pexpect() -> pexpect:
        """Import pexpect only when needed."""
        if platform.system() == "Windows":
            raise ValueError(
                "Persistent bash processes are not yet supported on Windows."
            )
        try:
            import pexpect
        except ImportError:
            raise ImportError(
                "pexpect required for persistent bash processes."
                " To install, run `pip install pexpect`."
            )
        return pexpect
    @staticmethod
    def _initialize_persistent_process(self: BashProcess, prompt: str) -> pexpect.spawn:
        # Start bash in a clean environment
        # Doesn't work on windows
        """
        Initializes a persistent bash setting in a
        clean environment.
        NOTE: Unavailable on Windows
        Args:
            Prompt(str): the bash command to execute
        """  # noqa: E501
        pexpect = self._lazy_import_pexpect()
        process = pexpect.spawn(
            "env", ["-i", "bash", "--norc", "--noprofile"], encoding="utf-8"
        )
        # Set the custom prompt
        process.sendline("PS1=" + prompt)
        process.expect_exact(prompt, timeout=10)
        return process
[docs]    def run(self, commands: Union[str, List[str]]) -> str:
        """
        Run commands in either an existing persistent
        subprocess or on in a new subprocess environment.
        Args:
            commands(List[str]): a list of commands to
                execute in the session
        """  # noqa: E501
execute in the session
        """  # noqa: E501
        if isinstance(commands, str):
            commands = [commands]
        commands = ";".join(commands)
        if self.process is not None:
            return self._run_persistent(
                commands,
            )
        else:
            return self._run(commands)
    def _run(self, command: str) -> str:
        """
        Runs a command in a subprocess and returns
        the output.
        Args:
            command: The command to run
        """  # noqa: E501
        try:
            output = subprocess.run(
                command,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
            ).stdout.decode()
        except subprocess.CalledProcessError as error:
            if self.return_err_output:
                return error.stdout.decode()
            return str(error)
        if self.strip_newlines:
            output = output.strip()
        return output
[docs]    def process_output(self, output: str, command: str) -> str:
        """
        Uses regex to remove the command from the output
        Args:
            output: a process' output string
            command: the executed command
        """  # noqa: E501
        pattern = re.escape(command) + r"\s*\n"
        output = re.sub(pattern, "", output, count=1)
        return output.strip()
    def _run_persistent(self, command: str) -> str:
        """
        Runs commands in a persistent environment
        and returns the output.
        Args:
            command: the command to execute
        """  # noqa: E501
        pexpect = self._lazy_import_pexpect()
        if self.process is None:
pexpect = self._lazy_import_pexpect()
        if self.process is None:
            raise ValueError("Process not initialized")
        self.process.sendline(command)
        # Clear the output with an empty string
        self.process.expect(self.prompt, timeout=10)
        self.process.sendline("")
        try:
            self.process.expect([self.prompt, pexpect.EOF], timeout=10)
        except pexpect.TIMEOUT:
            return f"Timeout error while executing command {command}"
        if self.process.after == pexpect.EOF:
            return f"Exited with error status: {self.process.exitstatus}"
        output = self.process.before
        output = self.process_output(output, command)
        if self.strip_newlines:
            return output.strip()
        return output
Source code for langchain_experimental.llm_bash.base
"""Chain that interprets a prompt and executes bash operations."""
from __future__ import annotations
import logging
import warnings
from typing import Any, Dict, List, Optional
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.schema import BasePromptTemplate, OutputParserException
from langchain.schema.language_model import BaseLanguageModel
from langchain_experimental.llm_bash.bash import BashProcess
from langchain_experimental.llm_bash.prompt import PROMPT
from langchain_experimental.pydantic_v1 import Extra, Field, root_validator
logger = logging.getLogger(__name__)
[docs]class LLMBashChain(Chain):
    """Chain that interprets a prompt and executes bash operations.
    Example:
        .. code-block:: python
            from langchain.chains import LLMBashChain
            from langchain.llms import OpenAI
            llm_bash = LLMBashChain.from_llm(OpenAI())
    """
    llm_chain: LLMChain
    llm: Optional[BaseLanguageModel] = None
    """[Deprecated] LLM wrapper to use."""
    input_key: str = "question"  #: :meta private:
    output_key: str = "answer"  #: :meta private:
    prompt: BasePromptTemplate = PROMPT
    """[Deprecated]"""
    bash_process: BashProcess = Field(default_factory=BashProcess)  #: :meta private:
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @root_validator(pre=True)
    def raise_deprecation(cls, values: Dict) -> Dict:
def raise_deprecation(cls, values: Dict) -> Dict:
        if "llm" in values:
            warnings.warn(
                "Directly instantiating an LLMBashChain with an llm is deprecated. "
                "Please instantiate with llm_chain or using the from_llm class method."
            )
            if "llm_chain" not in values and values["llm"] is not None:
                prompt = values.get("prompt", PROMPT)
                values["llm_chain"] = LLMChain(llm=values["llm"], prompt=prompt)
        return values
    # TODO: move away from `root_validator` since it is deprecated in pydantic v2
    #       and causes mypy type-checking failures (hence the `type: ignore`)
    @root_validator  # type: ignore[call-overload]
    def validate_prompt(cls, values: Dict) -> Dict:
        if values["llm_chain"].prompt.output_parser is None:
            raise ValueError(
                "The prompt used by llm_chain is expected to have an output_parser."
            )
        return values
    @property
    def input_keys(self) -> List[str]:
        """Expect input key.
        :meta private:
        """
        return [self.input_key]
    @property
    def output_keys(self) -> List[str]:
        """Expect output key.
        :meta private:
        """
        return [self.output_key]
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
_run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_key], verbose=self.verbose)
        t = self.llm_chain.predict(
            question=inputs[self.input_key], callbacks=_run_manager.get_child()
        )
        _run_manager.on_text(t, color="green", verbose=self.verbose)
        t = t.strip()
        try:
            parser = self.llm_chain.prompt.output_parser
            command_list = parser.parse(t)  # type: ignore[union-attr]
        except OutputParserException as e:
            _run_manager.on_chain_error(e, verbose=self.verbose)
            raise e
        if self.verbose:
            _run_manager.on_text("\nCode: ", verbose=self.verbose)
            _run_manager.on_text(
                str(command_list), color="yellow", verbose=self.verbose
            )
        output = self.bash_process.run(command_list)
        _run_manager.on_text("\nAnswer: ", verbose=self.verbose)
        _run_manager.on_text(output, color="yellow", verbose=self.verbose)
        return {self.output_key: output}
    @property
    def _chain_type(self) -> str:
        return "llm_bash_chain"
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: BasePromptTemplate = PROMPT,
        **kwargs: Any,
    ) -> LLMBashChain:
        llm_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(llm_chain=llm_chain, **kwargs)
Source code for langchain_experimental.llm_bash.prompt
# flake8: noqa
from __future__ import annotations
import re
from typing import List
from langchain.prompts.prompt import PromptTemplate
from langchain.schema import BaseOutputParser, OutputParserException
_PROMPT_TEMPLATE = """If someone asks you to perform a task, your job is to come up with a series of bash commands that will perform the task. There is no need to put "#!/bin/bash" in your answer. Make sure to reason step by step, using this format:
Question: "copy the files in the directory named 'target' into a new directory at the same level as target called 'myNewDirectory'"
I need to take the following actions:
- List all files in the directory
- Create a new directory
- Copy the files from the first directory into the second directory
```bash
ls
mkdir myNewDirectory
cp -r target/* myNewDirectory
```
That is the format. Begin!
Question: {question}"""
[docs]class BashOutputParser(BaseOutputParser):
    """Parser for bash output."""
[docs]    def parse(self, text: str) -> List[str]:
        if "```bash" in text:
            return self.get_code_blocks(text)
        else:
            raise OutputParserException(
                f"Failed to parse bash output. Got: {text}",
            )
[docs]    @staticmethod
    def get_code_blocks(t: str) -> List[str]:
        """Get multiple code blocks from the LLM result."""
        code_blocks: List[str] = []
        # Bash markdown code blocks
        pattern = re.compile(r"```bash(.*?)(?:\n\s*)```", re.DOTALL)
        for match in pattern.finditer(t):
for match in pattern.finditer(t):
            matched = match.group(1).strip()
            if matched:
                code_blocks.extend(
                    [line for line in matched.split("\n") if line.strip()]
                )
        return code_blocks
    @property
    def _type(self) -> str:
        return "bash"
PROMPT = PromptTemplate(
    input_variables=["question"],
    template=_PROMPT_TEMPLATE,
    output_parser=BashOutputParser(),
)
Source code for langchain_experimental.retrievers.vector_sql_database
"""Vector SQL Database Chain Retriever"""
from typing import Any, Dict, List
from langchain.callbacks.manager import (
    AsyncCallbackManagerForRetrieverRun,
    CallbackManagerForRetrieverRun,
)
from langchain.schema import BaseRetriever, Document
from langchain_experimental.sql.vector_sql import VectorSQLDatabaseChain
[docs]class VectorSQLDatabaseChainRetriever(BaseRetriever):
    """Retriever that uses SQLDatabase as Retriever"""
    sql_db_chain: VectorSQLDatabaseChain
    """SQL Database Chain"""
    page_content_key: str = "content"
    """column name for page content of documents"""
    def _get_relevant_documents(
        self,
        query: str,
        *,
        run_manager: CallbackManagerForRetrieverRun,
        **kwargs: Any,
    ) -> List[Document]:
        ret: List[Dict[str, Any]] = self.sql_db_chain(
            query, callbacks=run_manager.get_child(), **kwargs
        )["result"]
        return [
            Document(page_content=r[self.page_content_key], metadata=r) for r in ret
        ]
    async def _aget_relevant_documents(
        self, query: str, *, run_manager: AsyncCallbackManagerForRetrieverRun
    ) -> List[Document]:
        raise NotImplementedError
Source code for langchain_experimental.graph_transformers.diffbot
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
import requests
from langchain.graphs.graph_document import GraphDocument, Node, Relationship
from langchain.schema import Document
from langchain.utils import get_from_env
[docs]def format_property_key(s: str) -> str:
    words = s.split()
    if not words:
        return s
    first_word = words[0].lower()
    capitalized_words = [word.capitalize() for word in words[1:]]
    return "".join([first_word] + capitalized_words)
[docs]class NodesList:
    """
    Manages a list of nodes with associated properties.
    Attributes:
        nodes (Dict[Tuple, Any]): Stores nodes as keys and their properties as values.
            Each key is a tuple where the first element is the
            node ID and the second is the node type.
    """
[docs]    def __init__(self) -> None:
        self.nodes: Dict[Tuple[Union[str, int], str], Any] = dict()
[docs]    def add_node_property(
        self, node: Tuple[Union[str, int], str], properties: Dict[str, Any]
    ) -> None:
        """
        Adds or updates node properties.
        If the node does not exist in the list, it's added along with its properties.
        If the node already exists, its properties are updated with the new values.
        Args:
            node (Tuple): A tuple containing the node ID and node type.
            properties (Dict): A dictionary of properties to add or update for the node.
        """
        if node not in self.nodes:
            self.nodes[node] = properties
        else:
self.nodes[node] = properties
        else:
            self.nodes[node].update(properties)
[docs]    def return_node_list(self) -> List[Node]:
        """
        Returns the nodes as a list of Node objects.
        Each Node object will have its ID, type, and properties populated.
        Returns:
            List[Node]: A list of Node objects.
        """
        nodes = [
            Node(id=key[0], type=key[1], properties=self.nodes[key])
            for key in self.nodes
        ]
        return nodes
# Properties that should be treated as node properties instead of relationships
FACT_TO_PROPERTY_TYPE = [
    "Date",
    "Number",
    "Job title",
    "Cause of death",
    "Organization type",
    "Academic title",
]
schema_mapping = [
    ("HEADQUARTERS", "ORGANIZATION_LOCATIONS"),
    ("RESIDENCE", "PERSON_LOCATION"),
    ("ALL_PERSON_LOCATIONS", "PERSON_LOCATION"),
    ("CHILD", "HAS_CHILD"),
    ("PARENT", "HAS_PARENT"),
    ("CUSTOMERS", "HAS_CUSTOMER"),
    ("SKILLED_AT", "INTERESTED_IN"),
]
[docs]class SimplifiedSchema:
    """
    Provides functionality for working with a simplified schema mapping.
    Attributes:
        schema (Dict): A dictionary containing the mapping to simplified schema types.
    """
[docs]    def __init__(self) -> None:
        """Initializes the schema dictionary based on the predefined list."""
        self.schema = dict()
        for row in schema_mapping:
            self.schema[row[0]] = row[1]
[docs]    def get_type(self, type: str) -> str:
        """
[docs]    def get_type(self, type: str) -> str:
        """
        Retrieves the simplified schema type for a given original type.
        Args:
            type (str): The original schema type to find the simplified type for.
        Returns:
            str: The simplified schema type if it exists;
                 otherwise, returns the original type.
        """
        try:
            return self.schema[type]
        except KeyError:
            return type
[docs]class DiffbotGraphTransformer:
    """Transforms documents into graph documents using Diffbot's NLP API.
    A graph document transformation system takes a sequence of Documents and returns a
    sequence of Graph Documents.
    Example:
        .. code-block:: python
            class DiffbotGraphTransformer(BaseGraphDocumentTransformer):
                def transform_documents(
                    self, documents: Sequence[Document], **kwargs: Any
                ) -> Sequence[GraphDocument]:
                    results = []
                    for document in documents:
                        raw_results = self.nlp_request(document.page_content)
                        graph_document = self.process_response(raw_results, document)
                        results.append(graph_document)
                    return results
                async def atransform_documents(
                    self, documents: Sequence[Document], **kwargs: Any
                ) -> Sequence[Document]:
                    raise NotImplementedError
    """
[docs]    def __init__(
        self,
        diffbot_api_key: Optional[str] = None,
        fact_confidence_threshold: float = 0.7,
        include_qualifiers: bool = True,
        include_evidence: bool = True,
        simplified_schema: bool = True,
    ) -> None:
        """
        Initialize the graph transformer with various options.
        Args:
            diffbot_api_key (str):
Args:
            diffbot_api_key (str):
               The API key for Diffbot's NLP services.
            fact_confidence_threshold (float):
                Minimum confidence level for facts to be included.
            include_qualifiers (bool):
                Whether to include qualifiers in the relationships.
            include_evidence (bool):
                Whether to include evidence for the relationships.
            simplified_schema (bool):
                Whether to use a simplified schema for relationships.
        """
        self.diffbot_api_key = diffbot_api_key or get_from_env(
            "diffbot_api_key", "DIFFBOT_API_KEY"
        )
        self.fact_threshold_confidence = fact_confidence_threshold
        self.include_qualifiers = include_qualifiers
        self.include_evidence = include_evidence
        self.simplified_schema = None
        if simplified_schema:
            self.simplified_schema = SimplifiedSchema()
[docs]    def nlp_request(self, text: str) -> Dict[str, Any]:
        """
        Make an API request to the Diffbot NLP endpoint.
        Args:
            text (str): The text to be processed.
        Returns:
            Dict[str, Any]: The JSON response from the API.
        """
        # Relationship extraction only works for English
        payload = {
            "content": text,
            "lang": "en",
        }
        FIELDS = "facts"
        HOST = "nl.diffbot.com"
        url = (
            f"https://{HOST}/v1/?fields={FIELDS}&"
            f"token={self.diffbot_api_key}&language=en"
        )
        result = requests.post(url, data=payload)
        return result.json()
[docs]    def process_response(
        self, payload: Dict[str, Any], document: Document
self, payload: Dict[str, Any], document: Document
    ) -> GraphDocument:
        """
        Transform the Diffbot NLP response into a GraphDocument.
        Args:
            payload (Dict[str, Any]): The JSON response from Diffbot's NLP API.
            document (Document): The original document.
        Returns:
            GraphDocument: The transformed document as a graph.
        """
        # Return empty result if there are no facts
        if "facts" not in payload or not payload["facts"]:
            return GraphDocument(nodes=[], relationships=[], source=document)
        # Nodes are a custom class because we need to deduplicate
        nodes_list = NodesList()
        # Relationships are a list because we don't deduplicate nor anything else
        relationships = list()
        for record in payload["facts"]:
            # Skip if the fact is below the threshold confidence
            if record["confidence"] < self.fact_threshold_confidence:
                continue
            # TODO: It should probably be treated as a node property
            if not record["value"]["allTypes"]:
                continue
            # Define source node
            source_id = (
                record["entity"]["allUris"][0]
                if record["entity"]["allUris"]
                else record["entity"]["name"]
            )
            source_label = record["entity"]["allTypes"][0]["name"].capitalize()
            source_name = record["entity"]["name"]
            source_node = Node(id=source_id, type=source_label)
            nodes_list.add_node_property(
                (source_id, source_label), {"name": source_name}
            )
            # Define target node
            target_id = (
                record["value"]["allUris"][0]
                if record["value"]["allUris"]
if record["value"]["allUris"]
                else record["value"]["name"]
            )
            target_label = record["value"]["allTypes"][0]["name"].capitalize()
            target_name = record["value"]["name"]
            # Some facts are better suited as node properties
            if target_label in FACT_TO_PROPERTY_TYPE:
                nodes_list.add_node_property(
                    (source_id, source_label),
                    {format_property_key(record["property"]["name"]): target_name},
                )
            else:  # Define relationship
                # Define target node object
                target_node = Node(id=target_id, type=target_label)
                nodes_list.add_node_property(
                    (target_id, target_label), {"name": target_name}
                )
                # Define relationship type
                rel_type = record["property"]["name"].replace(" ", "_").upper()
                if self.simplified_schema:
                    rel_type = self.simplified_schema.get_type(rel_type)
                # Relationship qualifiers/properties
                rel_properties = dict()
                relationship_evidence = [el["passage"] for el in record["evidence"]][0]
                if self.include_evidence:
                    rel_properties.update({"evidence": relationship_evidence})
                if self.include_qualifiers and record.get("qualifiers"):
                    for property in record["qualifiers"]:
                        prop_key = format_property_key(property["property"]["name"])
                        rel_properties[prop_key] = property["value"]["name"]
                relationship = Relationship(
                    source=source_node,
                    target=target_node,
                    type=rel_type,
                    properties=rel_properties,
                )
                relationships.append(relationship)
        return GraphDocument(
            nodes=nodes_list.return_node_list(),
            relationships=relationships,
            source=document,
        )
relationships=relationships,
            source=document,
        )
[docs]    def convert_to_graph_documents(
        self, documents: Sequence[Document]
    ) -> List[GraphDocument]:
        """Convert a sequence of documents into graph documents.
        Args:
            documents (Sequence[Document]): The original documents.
            **kwargs: Additional keyword arguments.
        Returns:
            Sequence[GraphDocument]: The transformed documents as graphs.
        """
        results = []
        for document in documents:
            raw_results = self.nlp_request(document.page_content)
            graph_document = self.process_response(raw_results, document)
            results.append(graph_document)
        return results