Tony20100/langchain_full huggingface dataset:
"""Check that api answer prompt expects the right variables."""
            input_vars = values["api_answer_chain"].prompt.input_variables
            expected_vars = {"question", "api_docs", "api_url", "api_response"}
            if set(input_vars) != expected_vars:
                raise ValueError(
                    f"Input variables should be {expected_vars}, got {input_vars}"
                )
            return values
        def _call(
            self,
            inputs: Dict[str, Any],
            run_manager: Optional[CallbackManagerForChainRun] = None,
        ) -> Dict[str, str]:
            _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
            question = inputs[self.question_key]
            api_url = self.api_request_chain.predict(
                question=question,
                api_docs=self.api_docs,
                callbacks=_run_manager.get_child(),
            )
            _run_manager.on_text(api_url, color="green", end="\n", verbose=self.verbose)
            api_url = api_url.strip()
            if self.limit_to_domains and not _check_in_allowed_domain(
                api_url, self.limit_to_domains
            ):
                raise ValueError(
                    f"{api_url} is not in the allowed domains: {self.limit_to_domains}"
                )
            api_response = self.requests_wrapper.get(api_url)
            _run_manager.on_text(
                str(api_response), color="yellow", end="\n", verbose=self.verbose
            )
            answer = self.api_answer_chain.predict(
                question=question,
                api_docs=self.api_docs,
                api_url=api_url,
                api_response=api_response,
                callbacks=_run_manager.get_child(),
            )
            return {self.output_key: answer}
        async def _acall(
            self,
async def _acall(
            self,
            inputs: Dict[str, Any],
            run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
        ) -> Dict[str, str]:
            _run_manager = (
                run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
            )
            question = inputs[self.question_key]
            api_url = await self.api_request_chain.apredict(
                question=question,
                api_docs=self.api_docs,
                callbacks=_run_manager.get_child(),
            )
            await _run_manager.on_text(
                api_url, color="green", end="\n", verbose=self.verbose
            )
            api_url = api_url.strip()
            if self.limit_to_domains and not _check_in_allowed_domain(
                api_url, self.limit_to_domains
            ):
                raise ValueError(
                    f"{api_url} is not in the allowed domains: {self.limit_to_domains}"
                )
            api_response = await self.requests_wrapper.aget(api_url)
            await _run_manager.on_text(
                str(api_response), color="yellow", end="\n", verbose=self.verbose
            )
            answer = await self.api_answer_chain.apredict(
                question=question,
                api_docs=self.api_docs,
                api_url=api_url,
                api_response=api_response,
                callbacks=_run_manager.get_child(),
            )
            return {self.output_key: answer}
[docs]        @classmethod
        def from_llm_and_api_docs(
            cls,
            llm: BaseLanguageModel,
            api_docs: str,
            headers: Optional[dict] = None,
            api_url_prompt: BasePromptTemplate = API_URL_PROMPT,
            api_response_prompt: BasePromptTemplate = API_RESPONSE_PROMPT,
api_response_prompt: BasePromptTemplate = API_RESPONSE_PROMPT,
            limit_to_domains: Optional[Sequence[str]] = tuple(),
            **kwargs: Any,
        ) -> APIChain:
            """Load chain from just an LLM and the api docs."""
            get_request_chain = LLMChain(llm=llm, prompt=api_url_prompt)
            requests_wrapper = TextRequestsWrapper(headers=headers)
            get_answer_chain = LLMChain(llm=llm, prompt=api_response_prompt)
            return cls(
                api_request_chain=get_request_chain,
                api_answer_chain=get_answer_chain,
                requests_wrapper=requests_wrapper,
                api_docs=api_docs,
                limit_to_domains=limit_to_domains,
                **kwargs,
            )
        @property
        def _chain_type(self) -> str:
            return "api_chain"
except ImportError:
[docs]    class APIChain:  # type: ignore[no-redef]
        def __init__(self, *args: Any, **kwargs: Any) -> None:
            raise ImportError(
                "To use the APIChain, you must install the langchain_community package."
                "pip install langchain_community"
            )
links[i].removeAttribute("target");
		}
		"""
        self.page.evaluate(js)
        element = self.page_element_buffer.get(int(id))
        if element:
            x: float = element["center_x"]
            y: float = element["center_y"]
            self.page.mouse.click(x, y)
        else:
            print("Could not find element")  # noqa: T201
[docs]    def type(self, id: Union[str, int], text: str) -> None:
        self.click(id)
        self.page.keyboard.type(text)
[docs]    def enter(self) -> None:
        self.page.keyboard.press("Enter")
[docs]    def crawl(self) -> List[str]:
        page = self.page
        page_element_buffer = self.page_element_buffer
        start = time.time()
        page_state_as_text = []
        device_pixel_ratio: float = page.evaluate("window.devicePixelRatio")
        if platform == "darwin" and device_pixel_ratio == 1:  # lies
            device_pixel_ratio = 2
        win_upper_bound: float = page.evaluate("window.pageYOffset")
        win_left_bound: float = page.evaluate("window.pageXOffset")
        win_width: float = page.evaluate("window.screen.width")
        win_height: float = page.evaluate("window.screen.height")
        win_right_bound: float = win_left_bound + win_width
        win_lower_bound: float = win_upper_bound + win_height
        # 		percentage_progress_start = (win_upper_bound / document_scroll_height) * 100
        # 		percentage_progress_end = (
        # 			(win_height + win_upper_bound) / document_scroll_height
        # 		) * 100
        percentage_progress_start = 1
        percentage_progress_end = 2
percentage_progress_start = 1
        percentage_progress_end = 2
        page_state_as_text.append(
            {
                "x": 0,
                "y": 0,
                "text": "[scrollbar {:0.2f}-{:0.2f}%]".format(
                    round(percentage_progress_start, 2), round(percentage_progress_end)
                ),
            }
        )
        tree = self.client.send(
            "DOMSnapshot.captureSnapshot",
            {"computedStyles": [], "includeDOMRects": True, "includePaintOrder": True},
        )
        strings: Dict[int, str] = tree["strings"]
        document: Dict[str, Any] = tree["documents"][0]
        nodes: Dict[str, Any] = document["nodes"]
        backend_node_id: Dict[int, int] = nodes["backendNodeId"]
        attributes: Dict[int, Dict[int, Any]] = nodes["attributes"]
        node_value: Dict[int, int] = nodes["nodeValue"]
        parent: Dict[int, int] = nodes["parentIndex"]
        node_names: Dict[int, int] = nodes["nodeName"]
        is_clickable: Set[int] = set(nodes["isClickable"]["index"])
        input_value: Dict[str, Any] = nodes["inputValue"]
        input_value_index: List[int] = input_value["index"]
        input_value_values: List[int] = input_value["value"]
        layout: Dict[str, Any] = document["layout"]
        layout_node_index: List[int] = layout["nodeIndex"]
        bounds: Dict[int, List[float]] = layout["bounds"]
        cursor: int = 0
        child_nodes: Dict[str, List[Dict[str, Any]]] = {}
child_nodes: Dict[str, List[Dict[str, Any]]] = {}
        elements_in_view_port: List[ElementInViewPort] = []
        anchor_ancestry: Dict[str, Tuple[bool, Optional[int]]] = {"-1": (False, None)}
        button_ancestry: Dict[str, Tuple[bool, Optional[int]]] = {"-1": (False, None)}
        def convert_name(
            node_name: Optional[str], has_click_handler: Optional[bool]
        ) -> str:
            if node_name == "a":
                return "link"
            if node_name == "input":
                return "input"
            if node_name == "img":
                return "img"
            if (
                node_name == "button" or has_click_handler
            ):  # found pages that needed this quirk
                return "button"
            else:
                return "text"
        def find_attributes(
            attributes: Dict[int, Any], keys: List[str]
        ) -> Dict[str, str]:
            values = {}
            for [key_index, value_index] in zip(*(iter(attributes),) * 2):
                if value_index < 0:
                    continue
                key = strings[key_index]
                value = strings[value_index]
                if key in keys:
                    values[key] = value
                    keys.remove(key)
                    if not keys:
                        return values
            return values
        def add_to_hash_tree(
            hash_tree: Dict[str, Tuple[bool, Optional[int]]],
            tag: str,
            node_id: int,
            node_name: Optional[str],
            parent_id: int,
        ) -> Tuple[bool, Optional[int]]:
            parent_id_str = str(parent_id)
parent_id_str = str(parent_id)
            if not parent_id_str in hash_tree:
                parent_name = strings[node_names[parent_id]].lower()
                grand_parent_id = parent[parent_id]
                add_to_hash_tree(
                    hash_tree, tag, parent_id, parent_name, grand_parent_id
                )
            is_parent_desc_anchor, anchor_id = hash_tree[parent_id_str]
            # even if the anchor is nested in another anchor, we set the "root" for all descendants to be ::Self
            if node_name == tag:
                value: Tuple[bool, Optional[int]] = (True, node_id)
            elif (
                is_parent_desc_anchor
            ):  # reuse the parent's anchor_id (which could be much higher in the tree)
                value = (True, anchor_id)
            else:
                value = (
                    False,
                    None,
                )  # not a descendant of an anchor, most likely it will become text, an interactive element or discarded
            hash_tree[str(node_id)] = value
            return value
        for index, node_name_index in enumerate(node_names):
            node_parent = parent[index]
            node_name: Optional[str] = strings[node_name_index].lower()
            is_ancestor_of_anchor, anchor_id = add_to_hash_tree(
                anchor_ancestry, "a", index, node_name, node_parent
            )
            is_ancestor_of_button, button_id = add_to_hash_tree(
                button_ancestry, "button", index, node_name, node_parent
            )
            try:
                cursor = layout_node_index.index(
                    index
                )  # todo replace this with proper cursoring, ignoring the fact this is O(n^2) for the moment
            except:
                continue
except:
                continue
            if node_name in black_listed_elements:
                continue
            [x, y, width, height] = bounds[cursor]
            x /= device_pixel_ratio
            y /= device_pixel_ratio
            width /= device_pixel_ratio
            height /= device_pixel_ratio
            elem_left_bound = x
            elem_top_bound = y
            elem_right_bound = x + width
            elem_lower_bound = y + height
            partially_is_in_viewport = (
                elem_left_bound < win_right_bound
                and elem_right_bound >= win_left_bound
                and elem_top_bound < win_lower_bound
                and elem_lower_bound >= win_upper_bound
            )
            if not partially_is_in_viewport:
                continue
            meta_data: List[str] = []
            # inefficient to grab the same set of keys for kinds of objects, but it's fine for now
            element_attributes = find_attributes(
                attributes[index], ["type", "placeholder", "aria-label", "title", "alt"]
            )
            ancestor_exception = is_ancestor_of_anchor or is_ancestor_of_button
            ancestor_node_key = (
                None
                if not ancestor_exception
                else str(anchor_id)
                if is_ancestor_of_anchor
                else str(button_id)
            )
            ancestor_node = (
                None
                if not ancestor_exception
                else child_nodes.setdefault(str(ancestor_node_key), [])
            )
            if node_name == "#text" and ancestor_exception and ancestor_node:
                text = strings[node_value[index]]
                if text == "|" or text == "•":
                    continue
                ancestor_node.append({"type": "type", "value": text})
            else:
                if (
                    node_name == "input" and element_attributes.get("type") == "submit"
and converted_node_name != "link"
                and converted_node_name != "input"
                and converted_node_name != "img"
                and converted_node_name != "textarea"
            ) and inner_text.strip() == "":
                continue
            page_element_buffer[id_counter] = element
            if inner_text != "":
                elements_of_interest.append(
                    f"""<{converted_node_name} id={id_counter}{meta}>{inner_text}</{converted_node_name}>"""
                )
            else:
                elements_of_interest.append(
                    f"""<{converted_node_name} id={id_counter}{meta}/>"""
                )
            id_counter += 1
        print("Parsing time: {:0.2f} seconds".format(time.time() - start))  # noqa: T201
        return elements_of_interest
Source code for langchain.chains.natbot.base
"""Implement an LLM driven browser."""
from __future__ import annotations
import warnings
from typing import Any, Dict, List, Optional
from langchain_core.callbacks import CallbackManagerForChainRun
from langchain_core.language_models import BaseLanguageModel
from langchain_core.pydantic_v1 import Extra, root_validator
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.chains.natbot.prompt import PROMPT
[docs]class NatBotChain(Chain):
    """Implement an LLM driven browser.
    **Security Note**: This toolkit provides code to control a web-browser.
        The web-browser can be used to navigate to:
        - Any URL (including any internal network URLs)
        - And local files
        Exercise care if exposing this chain to end-users. Control who is able to
        access and use this chain, and isolate the network access of the server
        that hosts this chain.
        See https://python.langchain.com/docs/security for more information.
    Example:
        .. code-block:: python
            from langchain.chains import NatBotChain
            natbot = NatBotChain.from_default("Buy me a new hat.")
    """
    llm_chain: LLMChain
    objective: str
    """Objective that NatBot is tasked with completing."""
    llm: Optional[BaseLanguageModel] = None
    """[Deprecated] LLM wrapper to use."""
    input_url_key: str = "url"  #: :meta private:
    input_browser_content_key: str = "browser_content"  #: :meta private:
    previous_command: str = ""  #: :meta private:
    output_key: str = "command"  #: :meta private:
    class Config:
class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @root_validator(pre=True)
    def raise_deprecation(cls, values: Dict) -> Dict:
        if "llm" in values:
            warnings.warn(
                "Directly instantiating an NatBotChain with an llm is deprecated. "
                "Please instantiate with llm_chain argument or using the from_llm "
                "class method."
            )
            if "llm_chain" not in values and values["llm"] is not None:
                values["llm_chain"] = LLMChain(llm=values["llm"], prompt=PROMPT)
        return values
[docs]    @classmethod
    def from_default(cls, objective: str, **kwargs: Any) -> NatBotChain:
        """Load with default LLMChain."""
        raise NotImplementedError(
            "This method is no longer implemented. Please use from_llm."
            "llm = OpenAI(temperature=0.5, best_of=10, n=3, max_tokens=50)"
            "For example, NatBotChain.from_llm(llm, objective)"
        )
[docs]    @classmethod
    def from_llm(
        cls, llm: BaseLanguageModel, objective: str, **kwargs: Any
    ) -> NatBotChain:
        """Load from LLM."""
        llm_chain = LLMChain(llm=llm, prompt=PROMPT)
        return cls(llm_chain=llm_chain, objective=objective, **kwargs)
    @property
    def input_keys(self) -> List[str]:
        """Expect url and browser content.
        :meta private:
        """
Source code for langchain.chains.qa_generation.base
from __future__ import annotations
import json
from typing import Any, Dict, List, Optional
from langchain_core.callbacks import CallbackManagerForChainRun
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import BasePromptTemplate
from langchain_core.pydantic_v1 import Field
from langchain_text_splitters import RecursiveCharacterTextSplitter, TextSplitter
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.chains.qa_generation.prompt import PROMPT_SELECTOR
[docs]class QAGenerationChain(Chain):
    """Base class for question-answer generation chains."""
    llm_chain: LLMChain
    """LLM Chain that generates responses from user input and context."""
    text_splitter: TextSplitter = Field(
        default=RecursiveCharacterTextSplitter(chunk_overlap=500)
    )
    """Text splitter that splits the input into chunks."""
    input_key: str = "text"
    """Key of the input to the chain."""
    output_key: str = "questions"
    """Key of the output of the chain."""
    k: Optional[int] = None
    """Number of questions to generate."""
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: Optional[BasePromptTemplate] = None,
        **kwargs: Any,
    ) -> QAGenerationChain:
        """
        Create a QAGenerationChain from a language model.
        Args:
            llm: a language model
            prompt: a prompt template
            **kwargs: additional arguments
        Returns:
            a QAGenerationChain class
        """
Source code for langchain.chains.llm_math.base
"""Chain that interprets a prompt and executes python code to do math."""
from __future__ import annotations
import math
import re
import warnings
from typing import Any, Dict, List, Optional
from langchain_core.callbacks import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
)
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import BasePromptTemplate
from langchain_core.pydantic_v1 import Extra, root_validator
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.chains.llm_math.prompt import PROMPT
[docs]class LLMMathChain(Chain):
    """Chain that interprets a prompt and executes python code to do math.
    Example:
        .. code-block:: python
            from langchain.chains import LLMMathChain
            from langchain_community.llms import OpenAI
            llm_math = LLMMathChain.from_llm(OpenAI())
    """
    llm_chain: LLMChain
    llm: Optional[BaseLanguageModel] = None
    """[Deprecated] LLM wrapper to use."""
    prompt: BasePromptTemplate = PROMPT
    """[Deprecated] Prompt to use to translate to python if necessary."""
    input_key: str = "question"  #: :meta private:
    output_key: str = "answer"  #: :meta private:
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @root_validator(pre=True)
    def raise_deprecation(cls, values: Dict) -> Dict:
        try:
            import numexpr  # noqa: F401
        except ImportError:
try:
            import numexpr  # noqa: F401
        except ImportError:
            raise ImportError(
                "LLMMathChain requires the numexpr package. "
                "Please install it with `pip install numexpr`."
            )
        if "llm" in values:
            warnings.warn(
                "Directly instantiating an LLMMathChain with an llm is deprecated. "
                "Please instantiate with llm_chain argument or using the from_llm "
                "class method."
            )
            if "llm_chain" not in values and values["llm"] is not None:
                prompt = values.get("prompt", PROMPT)
                values["llm_chain"] = LLMChain(llm=values["llm"], prompt=prompt)
        return values
    @property
    def input_keys(self) -> List[str]:
        """Expect input key.
        :meta private:
        """
        return [self.input_key]
    @property
    def output_keys(self) -> List[str]:
        """Expect output key.
        :meta private:
        """
        return [self.output_key]
    def _evaluate_expression(self, expression: str) -> str:
        import numexpr
        try:
            local_dict = {"pi": math.pi, "e": math.e}
            output = str(
                numexpr.evaluate(
                    expression.strip(),
                    global_dict={},  # restrict access to globals
                    local_dict=local_dict,  # add common mathematical functions
                )
            )
        except Exception as e:
            raise ValueError(
                f'LLMMathChain._evaluate("{expression}") raised error: {e}.'
                " Please try again with a valid numerical expression"
            )
Source code for langchain.chains.conversational_retrieval.base
"""Chain for chatting with a vector database."""
from __future__ import annotations
import inspect
import warnings
from abc import abstractmethod
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
from langchain_core._api import deprecated
from langchain_core.callbacks import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
    Callbacks,
)
from langchain_core.documents import Document
from langchain_core.language_models import BaseLanguageModel
from langchain_core.messages import BaseMessage
from langchain_core.prompts import BasePromptTemplate
from langchain_core.pydantic_v1 import BaseModel, Extra, Field, root_validator
from langchain_core.retrievers import BaseRetriever
from langchain_core.runnables import RunnableConfig
from langchain_core.vectorstores import VectorStore
from langchain.chains.base import Chain
from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
from langchain.chains.combine_documents.stuff import StuffDocumentsChain
from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT
from langchain.chains.llm import LLMChain
from langchain.chains.question_answering import load_qa_chain
# Depending on the memory type and configuration, the chat history format may differ.
# This needs to be consolidated.
CHAT_TURN_TYPE = Union[Tuple[str, str], BaseMessage]
_ROLE_MAP = {"human": "Human: ", "ai": "Assistant: "}
def _get_chat_history(chat_history: List[CHAT_TURN_TYPE]) -> str:
    buffer = ""
    for dialogue_turn in chat_history:
        if isinstance(dialogue_turn, BaseMessage):