viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.embeddings.awa
from typing import Any, Dict, List
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, root_validator
[docs]class AwaEmbeddings(BaseModel, Embeddings):
    """Embedding documents and queries with Awa DB.
    Attributes:
        client: The AwaEmbedding client.
        model: The name of the model used for embedding.
         Default is "all-mpnet-base-v2".
    """
    client: Any  #: :meta private:
    model: str = "all-mpnet-base-v2"
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that awadb library is installed."""
        try:
            from awadb import AwaEmbedding
        except ImportError as exc:
            raise ImportError(
                "Could not import awadb library. "
                "Please install it with `pip install awadb`"
            ) from exc
        values["client"] = AwaEmbedding()
        return values
[docs]    def set_model(self, model_name: str) -> None:
        """Set the model used for embedding.
        The default model used is all-mpnet-base-v2
        Args:
            model_name: A string which represents the name of model.
        """
        self.model = model_name
        self.client.model_name = model_name
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed a list of documents using AwaEmbedding.
        Args:
            texts: The list of texts need to be embedded
        Returns:
            List of embeddings, one for each text.
        """
Returns:
            List of embeddings, one for each text.
        """
        return self.client.EmbeddingBatch(texts)
[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using AwaEmbedding.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        return self.client.Embedding(text)
Source code for langchain_community.embeddings.elasticsearch
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional
from langchain_core.utils import get_from_env
if TYPE_CHECKING:
    from elasticsearch import Elasticsearch
    from elasticsearch.client import MlClient
from langchain_core.embeddings import Embeddings
[docs]class ElasticsearchEmbeddings(Embeddings):
    """Elasticsearch embedding models.
    This class provides an interface to generate embeddings using a model deployed
    in an Elasticsearch cluster. It requires an Elasticsearch connection object
    and the model_id of the model deployed in the cluster.
    In Elasticsearch you need to have an embedding model loaded and deployed.
    - https://www.elastic.co/guide/en/elasticsearch/reference/current/infer-trained-model.html
    - https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-deploy-models.html
    """  # noqa: E501
[docs]    def __init__(
        self,
        client: MlClient,
        model_id: str,
        *,
        input_field: str = "text_field",
    ):
        """
        Initialize the ElasticsearchEmbeddings instance.
        Args:
            client (MlClient): An Elasticsearch ML client object.
            model_id (str): The model_id of the model deployed in the Elasticsearch
                cluster.
            input_field (str): The name of the key for the input text field in the
                document. Defaults to 'text_field'.
        """
        self.client = client
        self.model_id = model_id
        self.input_field = input_field
[docs]    @classmethod
    def from_credentials(
        cls,
        model_id: str,
        *,
        es_cloud_id: Optional[str] = None,
*,
        es_cloud_id: Optional[str] = None,
        es_user: Optional[str] = None,
        es_password: Optional[str] = None,
        input_field: str = "text_field",
    ) -> ElasticsearchEmbeddings:
        """Instantiate embeddings from Elasticsearch credentials.
        Args:
            model_id (str): The model_id of the model deployed in the Elasticsearch
                cluster.
            input_field (str): The name of the key for the input text field in the
                document. Defaults to 'text_field'.
            es_cloud_id: (str, optional): The Elasticsearch cloud ID to connect to.
            es_user: (str, optional): Elasticsearch username.
            es_password: (str, optional): Elasticsearch password.
        Example:
            .. code-block:: python
                from langchain_community.embeddings import ElasticsearchEmbeddings
                # Define the model ID and input field name (if different from default)
                model_id = "your_model_id"
                # Optional, only if different from 'text_field'
                input_field = "your_input_field"
                # Credentials can be passed in two ways. Either set the env vars
                # ES_CLOUD_ID, ES_USER, ES_PASSWORD and they will be automatically
                # pulled in, or pass them in directly as kwargs.
                embeddings = ElasticsearchEmbeddings.from_credentials(
                    model_id,
                    input_field=input_field,
                    # es_cloud_id="foo",
                    # es_user="bar",
                    # es_password="baz",
                )
                documents = [
                    "This is an example document.",
                    "Another example document to generate embeddings for.",
                ]
                embeddings_generator.embed_documents(documents)
        """
        try:
            from elasticsearch import Elasticsearch
            from elasticsearch.client import MlClient
from elasticsearch import Elasticsearch
            from elasticsearch.client import MlClient
        except ImportError:
            raise ImportError(
                "elasticsearch package not found, please install with 'pip install "
                "elasticsearch'"
            )
        es_cloud_id = es_cloud_id or get_from_env("es_cloud_id", "ES_CLOUD_ID")
        es_user = es_user or get_from_env("es_user", "ES_USER")
        es_password = es_password or get_from_env("es_password", "ES_PASSWORD")
        # Connect to Elasticsearch
        es_connection = Elasticsearch(
            cloud_id=es_cloud_id, basic_auth=(es_user, es_password)
        )
        client = MlClient(es_connection)
        return cls(client, model_id, input_field=input_field)
[docs]    @classmethod
    def from_es_connection(
        cls,
        model_id: str,
        es_connection: Elasticsearch,
        input_field: str = "text_field",
    ) -> ElasticsearchEmbeddings:
        """
        Instantiate embeddings from an existing Elasticsearch connection.
        This method provides a way to create an instance of the ElasticsearchEmbeddings
        class using an existing Elasticsearch connection. The connection object is used
        to create an MlClient, which is then used to initialize the
        ElasticsearchEmbeddings instance.
        Args:
        model_id (str): The model_id of the model deployed in the Elasticsearch cluster.
        es_connection (elasticsearch.Elasticsearch): An existing Elasticsearch
        connection object. input_field (str, optional): The name of the key for the
        input text field in the document. Defaults to 'text_field'.
        Returns:
        ElasticsearchEmbeddings: An instance of the ElasticsearchEmbeddings class.
        Example:
            .. code-block:: python
                from elasticsearch import Elasticsearch
Example:
            .. code-block:: python
                from elasticsearch import Elasticsearch
                from langchain_community.embeddings import ElasticsearchEmbeddings
                # Define the model ID and input field name (if different from default)
                model_id = "your_model_id"
                # Optional, only if different from 'text_field'
                input_field = "your_input_field"
                # Create Elasticsearch connection
                es_connection = Elasticsearch(
                    hosts=["localhost:9200"], http_auth=("user", "password")
                )
                # Instantiate ElasticsearchEmbeddings using the existing connection
                embeddings = ElasticsearchEmbeddings.from_es_connection(
                    model_id,
                    es_connection,
                    input_field=input_field,
                )
                documents = [
                    "This is an example document.",
                    "Another example document to generate embeddings for.",
                ]
                embeddings_generator.embed_documents(documents)
        """
        # Importing MlClient from elasticsearch.client within the method to
        # avoid unnecessary import if the method is not used
        from elasticsearch.client import MlClient
        # Create an MlClient from the given Elasticsearch connection
        client = MlClient(es_connection)
        # Return a new instance of the ElasticsearchEmbeddings class with
        # the MlClient, model_id, and input_field
        return cls(client, model_id, input_field=input_field)
    def _embedding_func(self, texts: List[str]) -> List[List[float]]:
        """
        Generate embeddings for the given texts using the Elasticsearch model.
        Args:
            texts (List[str]): A list of text strings to generate embeddings for.
        Returns:
            List[List[float]]: A list of embeddings, one for each text in the input
                list.
        """
        response = self.client.infer_trained_model(
list.
        """
        response = self.client.infer_trained_model(
            model_id=self.model_id, docs=[{self.input_field: text} for text in texts]
        )
        embeddings = [doc["predicted_value"] for doc in response["inference_results"]]
        return embeddings
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """
        Generate embeddings for a list of documents.
        Args:
            texts (List[str]): A list of document text strings to generate embeddings
                for.
        Returns:
            List[List[float]]: A list of embeddings, one for each document in the input
                list.
        """
        return self._embedding_func(texts)
[docs]    def embed_query(self, text: str) -> List[float]:
        """
        Generate an embedding for a single query text.
        Args:
            text (str): The query text to generate an embedding for.
        Returns:
            List[float]: The embedding for the input query text.
        """
        return self._embedding_func([text])[0]
Source code for langchain_community.embeddings.llamacpp
from typing import Any, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, Field, root_validator
[docs]class LlamaCppEmbeddings(BaseModel, Embeddings):
    """llama.cpp embedding models.
    To use, you should have the llama-cpp-python library installed, and provide the
    path to the Llama model as a named parameter to the constructor.
    Check out: https://github.com/abetlen/llama-cpp-python
    Example:
        .. code-block:: python
            from langchain_community.embeddings import LlamaCppEmbeddings
            llama = LlamaCppEmbeddings(model_path="/path/to/model.bin")
    """
    client: Any  #: :meta private:
    model_path: str
    n_ctx: int = Field(512, alias="n_ctx")
    """Token context window."""
    n_parts: int = Field(-1, alias="n_parts")
    """Number of parts to split the model into. 
    If -1, the number of parts is automatically determined."""
    seed: int = Field(-1, alias="seed")
    """Seed. If -1, a random seed is used."""
    f16_kv: bool = Field(False, alias="f16_kv")
    """Use half-precision for key/value cache."""
    logits_all: bool = Field(False, alias="logits_all")
    """Return logits for all tokens, not just the last token."""
    vocab_only: bool = Field(False, alias="vocab_only")
    """Only load the vocabulary, no weights."""
    use_mlock: bool = Field(False, alias="use_mlock")
    """Force system to keep model in RAM."""
"""Force system to keep model in RAM."""
    n_threads: Optional[int] = Field(None, alias="n_threads")
    """Number of threads to use. If None, the number 
    of threads is automatically determined."""
    n_batch: Optional[int] = Field(8, alias="n_batch")
    """Number of tokens to process in parallel.
    Should be a number between 1 and n_ctx."""
    n_gpu_layers: Optional[int] = Field(None, alias="n_gpu_layers")
    """Number of layers to be loaded into gpu memory. Default None."""
    verbose: bool = Field(True, alias="verbose")
    """Print verbose output to stderr."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that llama-cpp-python library is installed."""
        model_path = values["model_path"]
        model_param_names = [
            "n_ctx",
            "n_parts",
            "seed",
            "f16_kv",
            "logits_all",
            "vocab_only",
            "use_mlock",
            "n_threads",
            "n_batch",
            "verbose",
        ]
        model_params = {k: values[k] for k in model_param_names}
        # For backwards compatibility, only include if non-null.
        if values["n_gpu_layers"] is not None:
            model_params["n_gpu_layers"] = values["n_gpu_layers"]
        try:
            from llama_cpp import Llama
            values["client"] = Llama(model_path, embedding=True, **model_params)
        except ImportError:
            raise ModuleNotFoundError(
except ImportError:
            raise ModuleNotFoundError(
                "Could not import llama-cpp-python library. "
                "Please install the llama-cpp-python library to "
                "use this embedding model: pip install llama-cpp-python"
            )
        except Exception as e:
            raise ValueError(
                f"Could not load Llama model from path: {model_path}. "
                f"Received error {e}"
            )
        return values
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed a list of documents using the Llama model.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        embeddings = [self.client.embed(text) for text in texts]
        return [list(map(float, e)) for e in embeddings]
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a query using the Llama model.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        embedding = self.client.embed(text)
        return list(map(float, embedding))
Source code for langchain_community.embeddings.self_hosted
from typing import Any, Callable, List
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import Extra
from langchain_community.llms.self_hosted import SelfHostedPipeline
def _embed_documents(pipeline: Any, *args: Any, **kwargs: Any) -> List[List[float]]:
    """Inference function to send to the remote hardware.
    Accepts a sentence_transformer model_id and
    returns a list of embeddings for each document in the batch.
    """
    return pipeline(*args, **kwargs)
[docs]class SelfHostedEmbeddings(SelfHostedPipeline, Embeddings):
    """Custom embedding models on self-hosted remote hardware.
    Supported hardware includes auto-launched instances on AWS, GCP, Azure,
    and Lambda, as well as servers specified
    by IP address and SSH credentials (such as on-prem, or another
    cloud like Paperspace, Coreweave, etc.).
    To use, you should have the ``runhouse`` python package installed.
    Example using a model load function:
        .. code-block:: python
            from langchain_community.embeddings import SelfHostedEmbeddings
            from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
            import runhouse as rh
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            def get_pipeline():
                model_id = "facebook/bart-large"
                tokenizer = AutoTokenizer.from_pretrained(model_id)
                model = AutoModelForCausalLM.from_pretrained(model_id)
                return pipeline("feature-extraction", model=model, tokenizer=tokenizer)
            embeddings = SelfHostedEmbeddings(
                model_load_fn=get_pipeline,
                hardware=gpu
model_load_fn=get_pipeline,
                hardware=gpu
                model_reqs=["./", "torch", "transformers"],
            )
    Example passing in a pipeline path:
        .. code-block:: python
            from langchain_community.embeddings import SelfHostedHFEmbeddings
            import runhouse as rh
            from transformers import pipeline
            gpu = rh.cluster(name="rh-a10x", instance_type="A100:1")
            pipeline = pipeline(model="bert-base-uncased", task="feature-extraction")
            rh.blob(pickle.dumps(pipeline),
                path="models/pipeline.pkl").save().to(gpu, path="models")
            embeddings = SelfHostedHFEmbeddings.from_pipeline(
                pipeline="models/pipeline.pkl",
                hardware=gpu,
                model_reqs=["./", "torch", "transformers"],
            )
    """
    inference_fn: Callable = _embed_documents
    """Inference function to extract the embeddings on the remote hardware."""
    inference_kwargs: Any = None
    """Any kwargs to pass to the model's inference function."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Compute doc embeddings using a HuggingFace transformer model.
        Args:
            texts: The list of texts to embed.s
        Returns:
            List of embeddings, one for each text.
        """
        texts = list(map(lambda x: x.replace("\n", " "), texts))
        embeddings = self.client(self.pipeline_ref, texts)
        if not isinstance(embeddings, list):
            return embeddings.tolist()
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using a HuggingFace transformer model.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        text = text.replace("\n", " ")
        embeddings = self.client(self.pipeline_ref, text)
        if not isinstance(embeddings, list):
            return embeddings.tolist()
        return embeddings
Source code for langchain_community.embeddings.aleph_alpha
from typing import Any, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, root_validator
from langchain_core.utils import get_from_dict_or_env
[docs]class AlephAlphaAsymmetricSemanticEmbedding(BaseModel, Embeddings):
    """Aleph Alpha's asymmetric semantic embedding.
    AA provides you with an endpoint to embed a document and a query.
    The models were optimized to make the embeddings of documents and
    the query for a document as similar as possible.
    To learn more, check out: https://docs.aleph-alpha.com/docs/tasks/semantic_embed/
    Example:
        .. code-block:: python
            from aleph_alpha import AlephAlphaAsymmetricSemanticEmbedding
            embeddings = AlephAlphaAsymmetricSemanticEmbedding(
                normalize=True, compress_to_size=128
            )
            document = "This is a content of the document"
            query = "What is the content of the document?"
            doc_result = embeddings.embed_documents([document])
            query_result = embeddings.embed_query(query)
    """
    client: Any  #: :meta private:
    # Embedding params
    model: str = "luminous-base"
    """Model name to use."""
    compress_to_size: Optional[int] = None
    """Should the returned embeddings come back as an original 5120-dim vector, 
    or should it be compressed to 128-dim."""
    normalize: Optional[bool] = None
    """Should returned embeddings be normalized"""
    contextual_control_threshold: Optional[int] = None
    """Attention control parameters only apply to those tokens that have 
    explicitly been set in the request."""
    control_log_additive: bool = True
explicitly been set in the request."""
    control_log_additive: bool = True
    """Apply controls on prompt items by adding the log(control_factor) 
    to attention scores."""
    # Client params
    aleph_alpha_api_key: Optional[str] = None
    """API key for Aleph Alpha API."""
    host: str = "https://api.aleph-alpha.com"
    """The hostname of the API host. 
    The default one is "https://api.aleph-alpha.com")"""
    hosting: Optional[str] = None
    """Determines in which datacenters the request may be processed.
    You can either set the parameter to "aleph-alpha" or omit it (defaulting to None).
    Not setting this value, or setting it to None, gives us maximal flexibility 
    in processing your request in our
    own datacenters and on servers hosted with other providers. 
    Choose this option for maximal availability.
    Setting it to "aleph-alpha" allows us to only process the request 
    in our own datacenters.
    Choose this option for maximal data privacy."""
    request_timeout_seconds: int = 305
    """Client timeout that will be set for HTTP requests in the 
    `requests` library's API calls.
    Server will close all requests after 300 seconds with an internal server error."""
    total_retries: int = 8
    """The number of retries made in case requests fail with certain retryable 
    status codes. If the last
    retry fails a corresponding exception is raised. Note, that between retries 
    an exponential backoff
    is applied, starting with 0.5 s after the first retry and doubling for each 
    retry made. So with the
retry made. So with the
    default setting of 8 retries a total wait time of 63.5 s is added between 
    the retries."""
    nice: bool = False
    """Setting this to True, will signal to the API that you intend to be 
    nice to other users
    by de-prioritizing your request below concurrent ones."""
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        aleph_alpha_api_key = get_from_dict_or_env(
            values, "aleph_alpha_api_key", "ALEPH_ALPHA_API_KEY"
        )
        try:
            from aleph_alpha_client import Client
            values["client"] = Client(
                token=aleph_alpha_api_key,
                host=values["host"],
                hosting=values["hosting"],
                request_timeout_seconds=values["request_timeout_seconds"],
                total_retries=values["total_retries"],
                nice=values["nice"],
            )
        except ImportError:
            raise ValueError(
                "Could not import aleph_alpha_client python package. "
                "Please install it with `pip install aleph_alpha_client`."
            )
        return values
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Call out to Aleph Alpha's asymmetric Document endpoint.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        try:
            from aleph_alpha_client import (
                Prompt,
                SemanticEmbeddingRequest,
                SemanticRepresentation,
            )
        except ImportError:
            raise ValueError(