Tony20100/langchain_full huggingface dataset:
langchain_core.beta.runnables.context.Context¶
class langchain_core.beta.runnables.context.Context[source]¶
Context for a runnable.
The Context class provides methods for creating context scopes,
getters, and setters within a runnable. It allows for managing
and accessing contextual information throughout the execution
of a program.
Example
from langchain_core.beta.runnables.context import Context
from langchain_core.runnables.passthrough import RunnablePassthrough
from langchain_core.prompts.prompt import PromptTemplate
from langchain_core.output_parsers.string import StrOutputParser
from tests.unit_tests.fake.llm import FakeListLLM
chain = (
    Context.setter("input")
    | {
        "context": RunnablePassthrough()
                | Context.setter("context"),
        "question": RunnablePassthrough(),
    }
    | PromptTemplate.from_template("{context} {question}")
    | FakeListLLM(responses=["hello"])
    | StrOutputParser()
    | {
        "result": RunnablePassthrough(),
        "context": Context.getter("context"),
        "input": Context.getter("input"),
    }
)
# Use the chain
output = chain.invoke("What's your name?")
print(output["result"])  # Output: "hello"
print(output["context"])  # Output: "What's your name?"
print(output["input"])  # Output: "What's your name?
Methods
__init__()
create_scope(scope, /)
Create a context scope.
getter(key, /)
setter([_key, _value])
__init__()¶
static create_scope(scope: str, /) → PrefixContext[source]¶
Create a context scope.
Parameters
scope (str) – The scope.
Returns
The context scope.
Return type
PrefixContext
Returns
The context scope.
Return type
PrefixContext
static getter(key: Union[str, List[str]], /) → ContextGet[source]¶
Parameters
key (Union[str, List[str]]) – 
Return type
ContextGet
static setter(_key: Optional[str] = None, _value: Optional[Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]] = None, /, **kwargs: Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]) → ContextSet[source]¶
Parameters
_key (Optional[str]) – 
_value (Optional[Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]]) – 
kwargs (Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]) – 
Return type
ContextSet
langchain_core.beta.runnables.context.PrefixContext¶
class langchain_core.beta.runnables.context.PrefixContext(prefix: str = '')[source]¶
Context for a runnable with a prefix.
Attributes
prefix
Methods
__init__([prefix])
getter(key, /)
setter([_key, _value])
Parameters
prefix (str) – 
__init__(prefix: str = '')[source]¶
Parameters
prefix (str) – 
getter(key: Union[str, List[str]], /) → ContextGet[source]¶
Parameters
key (Union[str, List[str]]) – 
Return type
ContextGet
setter(_key: Optional[str] = None, _value: Optional[Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]] = None, /, **kwargs: Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]) → ContextSet[source]¶
Parameters
_key (Optional[str]) – 
_value (Optional[Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]]) – 
kwargs (Union[Runnable[Input, Output], Callable[[Input], Output], Callable[[Input], Awaitable[Output]], Any]) – 
Return type
ContextSet
langchain_core.beta.runnables.context.ContextGet¶
class langchain_core.beta.runnables.context.ContextGet[source]¶
Bases: RunnableSerializable
[Beta] Get a context value.
Notes
Create a new model by parsing and validating input data from keyword arguments.
Raises ValidationError if the input data cannot be parsed to form a valid model.
param key: Union[str, List[str]] [Required]¶
param prefix: str = ''¶
async abatch(inputs: List[Input], config: Optional[Union[RunnableConfig, List[RunnableConfig]]] = None, *, return_exceptions: bool = False, **kwargs: Optional[Any]) → List[Output]¶
Default implementation runs ainvoke in parallel using asyncio.gather.
The default implementation of batch works well for IO bound runnables.
Subclasses should override this method if they can batch more efficiently;
e.g., if the underlying runnable uses an API which supports a batch mode.
Parameters
inputs (List[Input]) – 
config (Optional[Union[RunnableConfig, List[RunnableConfig]]]) – 
return_exceptions (bool) – 
kwargs (Optional[Any]) – 
Return type
List[Output]
async abatch_as_completed(inputs: Sequence[Input], config: Optional[Union[RunnableConfig, Sequence[RunnableConfig]]] = None, *, return_exceptions: bool = False, **kwargs: Optional[Any]) → AsyncIterator[Tuple[int, Union[Output, Exception]]]¶
Run ainvoke in parallel on a list of inputs,
yielding results as they complete.
Parameters
inputs (Sequence[Input]) – 
config (Optional[Union[RunnableConfig, Sequence[RunnableConfig]]]) – 
return_exceptions (bool) – 
kwargs (Optional[Any]) – 
Return type
AsyncIterator[Tuple[int, Union[Output, Exception]]]
langchain_text_splitters.character.RecursiveCharacterTextSplitter¶
class langchain_text_splitters.character.RecursiveCharacterTextSplitter(separators: Optional[List[str]] = None, keep_separator: bool = True, is_separator_regex: bool = False, **kwargs: Any)[source]¶
Splitting text by recursively look at characters.
Recursively tries to split by different characters to find one
that works.
Create a new TextSplitter.
Methods
__init__([separators, keep_separator, ...])
Create a new TextSplitter.
atransform_documents(documents, **kwargs)
Asynchronously transform a list of documents.
create_documents(texts[, metadatas])
Create documents from a list of texts.
from_huggingface_tokenizer(tokenizer, **kwargs)
Text splitter that uses HuggingFace tokenizer to count length.
from_language(language, **kwargs)
from_tiktoken_encoder([encoding_name, ...])
Text splitter that uses tiktoken encoder to count length.
get_separators_for_language(language)
split_documents(documents)
Split documents.
split_text(text)
Split text into multiple components.
transform_documents(documents, **kwargs)
Transform sequence of documents by splitting them.
Parameters
separators (Optional[List[str]]) – 
keep_separator (bool) – 
is_separator_regex (bool) – 
kwargs (Any) – 
__init__(separators: Optional[List[str]] = None, keep_separator: bool = True, is_separator_regex: bool = False, **kwargs: Any) → None[source]¶
Create a new TextSplitter.
Parameters
separators (Optional[List[str]]) – 
keep_separator (bool) – 
is_separator_regex (bool) – 
kwargs (Any) – 
Return type
is_separator_regex (bool) – 
kwargs (Any) – 
Return type
None
async atransform_documents(documents: Sequence[Document], **kwargs: Any) → Sequence[Document]¶
Asynchronously transform a list of documents.
Parameters
documents (Sequence[Document]) – A sequence of Documents to be transformed.
kwargs (Any) – 
Returns
A list of transformed Documents.
Return type
Sequence[Document]
create_documents(texts: List[str], metadatas: Optional[List[dict]] = None) → List[Document]¶
Create documents from a list of texts.
Parameters
texts (List[str]) – 
metadatas (Optional[List[dict]]) – 
Return type
List[Document]
classmethod from_huggingface_tokenizer(tokenizer: Any, **kwargs: Any) → TextSplitter¶
Text splitter that uses HuggingFace tokenizer to count length.
Parameters
tokenizer (Any) – 
kwargs (Any) – 
Return type
TextSplitter
classmethod from_language(language: Language, **kwargs: Any) → RecursiveCharacterTextSplitter[source]¶
Parameters
language (Language) – 
kwargs (Any) – 
Return type
RecursiveCharacterTextSplitter
classmethod from_tiktoken_encoder(encoding_name: str = 'gpt2', model_name: Optional[str] = None, allowed_special: Union[Literal['all'], AbstractSet[str]] = {}, disallowed_special: Union[Literal['all'], Collection[str]] = 'all', **kwargs: Any) → TS¶
Text splitter that uses tiktoken encoder to count length.
Parameters
encoding_name (str) – 
model_name (Optional[str]) – 
allowed_special (Union[Literal['all'], ~typing.AbstractSet[str]]) – 
disallowed_special (Union[Literal['all'], ~typing.Collection[str]]) –
disallowed_special (Union[Literal['all'], ~typing.Collection[str]]) – 
kwargs (Any) – 
Return type
TS
static get_separators_for_language(language: Language) → List[str][source]¶
Parameters
language (Language) – 
Return type
List[str]
split_documents(documents: Iterable[Document]) → List[Document]¶
Split documents.
Parameters
documents (Iterable[Document]) – 
Return type
List[Document]
split_text(text: str) → List[str][source]¶
Split text into multiple components.
Parameters
text (str) – 
Return type
List[str]
transform_documents(documents: Sequence[Document], **kwargs: Any) → Sequence[Document]¶
Transform sequence of documents by splitting them.
Parameters
documents (Sequence[Document]) – 
kwargs (Any) – 
Return type
Sequence[Document]
Examples using RecursiveCharacterTextSplitter¶
Activeloop Deep Memory
Apache Cassandra
Astra DB
Build a Local RAG Application
Build a Query Analysis System
Build a Retrieval Augmented Generation (RAG) App
Build an Agent
Cohere reranker
Conversational RAG
Cross Encoder Reranker
FlashRank reranker
Google Cloud Vertex AI Reranker
Google Vertex AI Vector Search
How to add chat history
How to add retrieval to chatbots
How to add scores to retriever results
How to get a RAG application to add citations
How to get your RAG application to return sources
How to handle cases where no queries are generated
How to handle multiple queries when doing query analysis
How to handle multiple retrievers when doing query analysis
How to recursively split text by characters
How to retrieve using multiple vectors per document
How to split Markdown by Headers
How to split by HTML header
How to split Markdown by Headers
How to split by HTML header 
How to split by HTML sections
How to split code
How to split text by tokens 
How to stream results from your RAG application
How to use the MultiQueryRetriever
How to use the Parent Document Retriever
Jina Reranker
LLMLingua Document Compressor
Maritalk
MongoDB Atlas
OpenVINO Reranker
RAGatouille
RankLLM Reranker
RePhraseQuery
Source Code
UpTrain
Vearch
VoyageAI Reranker
Yellowbrick
YouTube audio
Zep
Zep Cloud
viking DB
langchain_text_splitters.character.CharacterTextSplitter¶
class langchain_text_splitters.character.CharacterTextSplitter(separator: str = '\n\n', is_separator_regex: bool = False, **kwargs: Any)[source]¶
Splitting text that looks at characters.
Create a new TextSplitter.
Methods
__init__([separator, is_separator_regex])
Create a new TextSplitter.
atransform_documents(documents, **kwargs)
Asynchronously transform a list of documents.
create_documents(texts[, metadatas])
Create documents from a list of texts.
from_huggingface_tokenizer(tokenizer, **kwargs)
Text splitter that uses HuggingFace tokenizer to count length.
from_tiktoken_encoder([encoding_name, ...])
Text splitter that uses tiktoken encoder to count length.
split_documents(documents)
Split documents.
split_text(text)
Split incoming text and return chunks.
transform_documents(documents, **kwargs)
Transform sequence of documents by splitting them.
Parameters
separator (str) – 
is_separator_regex (bool) – 
kwargs (Any) – 
__init__(separator: str = '\n\n', is_separator_regex: bool = False, **kwargs: Any) → None[source]¶
Create a new TextSplitter.
Parameters
separator (str) – 
is_separator_regex (bool) – 
kwargs (Any) – 
Return type
None
async atransform_documents(documents: Sequence[Document], **kwargs: Any) → Sequence[Document]¶
Asynchronously transform a list of documents.
Parameters
documents (Sequence[Document]) – A sequence of Documents to be transformed.
kwargs (Any) – 
Returns
A list of transformed Documents.
Return type
Sequence[Document]
Transform sequence of documents by splitting them.
Parameters
documents (Sequence[Document]) – 
kwargs (Any) – 
Return type
Sequence[Document]
Examples using CharacterTextSplitter¶
Activeloop Deep Lake
Alibaba Cloud OpenSearch
Amazon Document DB
AnalyticDB
Annoy
AwaDB
Azure AI Search
Azure Cosmos DB
Bagel
BagelDB
Baidu Cloud ElasticSearch VectorSearch
Baidu VectorDB
Caching
China Mobile ECloud ElasticSearch VectorSearch
Chroma
Clarifai
ClickHouse
Confident
Couchbase 
DashVector
Databricks Vector Search
DingoDB
DocArray HnswSearch
DocArray InMemorySearch
Document Comparison
DuckDB
Elasticsearch
Epsilla
Faiss
Faiss (Async)
Google Memorystore for Redis
Hippo
Hologres
How to create and query vector stores
How to do retrieval with contextual compression
How to handle long text when doing extraction
How to split by character
How to split text by tokens 
How to use a vectorstore as a retriever
How to use the LangChain indexing API
Intel’s Visual Data Management System (VDMS)
Jaguar Vector Database
JaguarDB Vector Database
Kinetica Vectorstore API
Kinetica Vectorstore based Retriever
LanceDB
Lantern
Manifest
ManticoreSearch VectorStore
Marqo
Meilisearch
Milvus
Model caches
Momento Vector Index (MVI)
MyScale
Neo4j Vector Index
OpenAI
OpenSearch
PGVecto.rs
Pinecone
Postgres Embedding
Psychic
Qdrant
Relyt
Rockset
SAP HANA Cloud Vector Engine
SQLite-VSS
ScaNN
SemaDB
SAP HANA Cloud Vector Engine
SQLite-VSS
ScaNN
SemaDB
SingleStoreDB
Summarize Text
Supabase (Postgres)
SurrealDB
Tair
Tencent Cloud VectorDB
TiDB Vector
Tigris
TileDB
Timescale Vector (Postgres)
Typesense
USearch
Upstash Vector
VDMS
Vald
Vectara
Vectara 
Vespa
Weaviate
Xata
Zilliz
scikit-learn
vlite
Source code for langchain.hub
"""Interface with the LangChain Hub."""
from __future__ import annotations
import json
from typing import TYPE_CHECKING, Any, Optional
from langchain_core.load.dump import dumps
from langchain_core.load.load import loads
from langchain_core.prompts import BasePromptTemplate
if TYPE_CHECKING:
    from langchainhub import Client
def _get_client(api_url: Optional[str] = None, api_key: Optional[str] = None) -> Client:
    try:
        from langchainhub import Client
    except ImportError as e:
        raise ImportError(
            "Could not import langchainhub, please install with `pip install "
            "langchainhub`."
        ) from e
    # Client logic will also attempt to load URL/key from environment variables
    return Client(api_url, api_key=api_key)
[docs]def push(
    repo_full_name: str,
    object: Any,
    *,
    api_url: Optional[str] = None,
    api_key: Optional[str] = None,
    parent_commit_hash: Optional[str] = "latest",
    new_repo_is_public: bool = True,
    new_repo_description: str = "",
) -> str:
    """
    Push an object to the hub and returns the URL it can be viewed at in a browser.
    :param repo_full_name: The full name of the repo to push to in the format of
        `owner/repo`.
    :param object: The LangChain to serialize and push to the hub.
    :param api_url: The URL of the LangChain Hub API. Defaults to the hosted API service
        if you have an api key set, or a localhost instance if not.
if you have an api key set, or a localhost instance if not.
    :param api_key: The API key to use to authenticate with the LangChain Hub API.
    :param parent_commit_hash: The commit hash of the parent commit to push to. Defaults
        to the latest commit automatically.
    :param new_repo_is_public: Whether the repo should be public. Defaults to
        True (Public by default).
    :param new_repo_description: The description of the repo. Defaults to an empty
        string.
    """
    client = _get_client(api_url=api_url, api_key=api_key)
    manifest_json = dumps(object)
    message = client.push(
        repo_full_name,
        manifest_json,
        parent_commit_hash=parent_commit_hash,
        new_repo_is_public=new_repo_is_public,
        new_repo_description=new_repo_description,
    )
    return message
[docs]def pull(
    owner_repo_commit: str,
    *,
    api_url: Optional[str] = None,
    api_key: Optional[str] = None,
) -> Any:
    """
    Pull an object from the hub and returns it as a LangChain object.
    :param owner_repo_commit: The full name of the repo to pull from in the format of
        `owner/repo:commit_hash`.
    :param api_url: The URL of the LangChain Hub API. Defaults to the hosted API service
        if you have an api key set, or a localhost instance if not.
    :param api_key: The API key to use to authenticate with the LangChain Hub API.
    """
    client = _get_client(api_url=api_url, api_key=api_key)
    if hasattr(client, "pull_repo"):
        # >= 0.1.15
if hasattr(client, "pull_repo"):
        # >= 0.1.15
        res_dict = client.pull_repo(owner_repo_commit)
        obj = loads(json.dumps(res_dict["manifest"]))
        if isinstance(obj, BasePromptTemplate):
            if obj.metadata is None:
                obj.metadata = {}
            obj.metadata["lc_hub_owner"] = res_dict["owner"]
            obj.metadata["lc_hub_repo"] = res_dict["repo"]
            obj.metadata["lc_hub_commit_hash"] = res_dict["commit_hash"]
        return obj
    # Then it's < 0.1.15
    resp: str = client.pull(owner_repo_commit)
    return loads(resp)
self.chain_colors = get_color_mapping(chain_range)
        self.names = names
[docs]    @classmethod
    def from_llms(
        cls, llms: List[BaseLLM], prompt: Optional[PromptTemplate] = None
    ) -> ModelLaboratory:
        """Initialize with LLMs to experiment with and optional prompt.
        Args:
            llms: list of LLMs to experiment with
            prompt: Optional prompt to use to prompt the LLMs. Defaults to None.
                If a prompt was provided, it should only have one input variable.
        """
        if prompt is None:
            prompt = PromptTemplate(input_variables=["_input"], template="{_input}")
        chains = [LLMChain(llm=llm, prompt=prompt) for llm in llms]
        names = [str(llm) for llm in llms]
        return cls(chains, names=names)
[docs]    def compare(self, text: str) -> None:
        """Compare model outputs on an input text.
        If a prompt was provided with starting the laboratory, then this text will be
        fed into the prompt. If no prompt was provided, then the input text is the
        entire prompt.
        Args:
            text: input text to run all models on.
        """
        print(f"\033[1mInput:\033[0m\n{text}\n")  # noqa: T201
        for i, chain in enumerate(self.chains):
            if self.names is not None:
                name = self.names[i]
            else:
                name = str(chain)
            print_text(name, end="\n")
            output = chain.run(text)
            print_text(output, color=self.chain_colors[str(i)], end="\n\n")
Source code for langchain.callbacks.streaming_aiter
from __future__ import annotations
import asyncio
from typing import Any, AsyncIterator, Dict, List, Literal, Union, cast
from langchain_core.callbacks import AsyncCallbackHandler
from langchain_core.outputs import LLMResult
# TODO If used by two LLM runs in parallel this won't work as expected
[docs]class AsyncIteratorCallbackHandler(AsyncCallbackHandler):
    """Callback handler that returns an async iterator."""
    queue: asyncio.Queue[str]
    done: asyncio.Event
    @property
    def always_verbose(self) -> bool:
        return True
[docs]    def __init__(self) -> None:
        self.queue = asyncio.Queue()
        self.done = asyncio.Event()
[docs]    async def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        # If two calls are made in a row, this resets the state
        self.done.clear()
[docs]    async def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        if token is not None and token != "":
            self.queue.put_nowait(token)
[docs]    async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        self.done.set()
[docs]    async def on_llm_error(self, error: BaseException, **kwargs: Any) -> None:
        self.done.set()
    # TODO implement the other methods
[docs]    async def aiter(self) -> AsyncIterator[str]:
        while not self.queue.empty() or not self.done.is_set():
            # Wait for the next token in the queue,
            # but stop waiting if the done event is set
Source code for langchain.callbacks.streaming_stdout_final_only
"""Callback Handler streams to stdout on new llm token."""
import sys
from typing import Any, Dict, List, Optional
from langchain_core.callbacks import StreamingStdOutCallbackHandler
DEFAULT_ANSWER_PREFIX_TOKENS = ["Final", "Answer", ":"]
[docs]class FinalStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler):
    """Callback handler for streaming in agents.
    Only works with agents using LLMs that support streaming.
    Only the final output of the agent will be streamed.
    """
[docs]    def append_to_last_tokens(self, token: str) -> None:
        self.last_tokens.append(token)
        self.last_tokens_stripped.append(token.strip())
        if len(self.last_tokens) > len(self.answer_prefix_tokens):
            self.last_tokens.pop(0)
            self.last_tokens_stripped.pop(0)
[docs]    def check_if_answer_reached(self) -> bool:
        if self.strip_tokens:
            return self.last_tokens_stripped == self.answer_prefix_tokens_stripped
        else:
            return self.last_tokens == self.answer_prefix_tokens
[docs]    def __init__(
        self,
        *,
        answer_prefix_tokens: Optional[List[str]] = None,
        strip_tokens: bool = True,
        stream_prefix: bool = False,
    ) -> None:
        """Instantiate FinalStreamingStdOutCallbackHandler.
        Args:
            answer_prefix_tokens: Token sequence that prefixes the answer.
                Default is ["Final", "Answer", ":"]
            strip_tokens: Ignore white spaces and new lines when comparing
                answer_prefix_tokens to last tokens? (to determine if answer has been
                reached)
            stream_prefix: Should answer prefix itself also be streamed?
        """
Source code for langchain.callbacks.tracers.logging
__all__ = ["LoggingCallbackHandler"]
import logging
from typing import Any, Optional
from uuid import UUID
from langchain_core.exceptions import TracerException
from langchain_core.tracers.stdout import FunctionCallbackHandler
from langchain_core.utils.input import get_bolded_text, get_colored_text
[docs]class LoggingCallbackHandler(FunctionCallbackHandler):
    """Tracer that logs via the input Logger."""
    name: str = "logging_callback_handler"
[docs]    def __init__(
        self,
        logger: logging.Logger,
        log_level: int = logging.INFO,
        extra: Optional[dict] = None,
        **kwargs: Any,
    ) -> None:
        log_method = getattr(logger, logging.getLevelName(level=log_level).lower())
        def callback(text: str) -> None:
            log_method(text, extra=extra)
        super().__init__(function=callback, **kwargs)
[docs]    def on_text(
        self,
        text: str,
        *,
        run_id: UUID,
        parent_run_id: Optional[UUID] = None,
        **kwargs: Any,
    ) -> None:
        try:
            crumbs_str = f"[{self.get_breadcrumbs(run=self._get_run(run_id=run_id))}] "
        except TracerException:
            crumbs_str = ""
        self.function_callback(
            f'{get_colored_text("[text]", color="blue")}'
            f' {get_bolded_text(f"{crumbs_str}New text:")}\n{text}'
        )