viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.embeddings.bookend
"""Wrapper around Bookend AI embedding models."""
import json
from typing import Any, List
import requests
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Field
API_URL = "https://api.bookend.ai/"
DEFAULT_TASK = "embeddings"
PATH = "/models/predict"
[docs]class BookendEmbeddings(BaseModel, Embeddings):
    """Bookend AI sentence_transformers embedding models.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import BookendEmbeddings
            bookend = BookendEmbeddings(
                domain={domain}
                api_token={api_token}
                model_id={model_id}
            )
            bookend.embed_documents([
                "Please put on these earmuffs because I can't you hear.",
                "Baby wipes are made of chocolate stardust.",
            ])
            bookend.embed_query(
                "She only paints with bold colors; she does not like pastels."
            )
    """
    domain: str
    """Request for a domain at https://bookend.ai/ to use this embeddings module."""
    api_token: str
    """Request for an API token at https://bookend.ai/ to use this embeddings module."""
    model_id: str
    """Embeddings model ID to use."""
    auth_header: dict = Field(default_factory=dict)
    def __init__(self, **kwargs: Any):
        super().__init__(**kwargs)
        self.auth_header = {"Authorization": "Basic {}".format(self.api_token)}
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed documents using a Bookend deployed embeddings model.
        Args:
"""Embed documents using a Bookend deployed embeddings model.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        result = []
        headers = self.auth_header
        headers["Content-Type"] = "application/json; charset=utf-8"
        params = {
            "model_id": self.model_id,
            "task": DEFAULT_TASK,
        }
        for text in texts:
            data = json.dumps(
                {"text": text, "question": None, "context": None, "instruction": None}
            )
            r = requests.request(
                "POST",
                API_URL + self.domain + PATH,
                headers=headers,
                params=params,
                data=data,
            )
            result.append(r.json()[0]["data"])
        return result
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a query using a Bookend deployed embeddings model.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        return self.embed_documents([text])[0]
Source code for langchain_community.embeddings.llm_rails
""" This file is for LLMRails Embedding """
import logging
import os
from typing import List, Optional
import requests
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra
[docs]class LLMRailsEmbeddings(BaseModel, Embeddings):
    """LLMRails embedding models.
    To use, you should have the  environment
    variable ``LLM_RAILS_API_KEY`` set with your API key or pass it
    as a named parameter to the constructor.
    Model can be one of ["embedding-english-v1","embedding-multi-v1"]
    Example:
        .. code-block:: python
            from langchain_community.embeddings import LLMRailsEmbeddings
            cohere = LLMRailsEmbeddings(
                model="embedding-english-v1", api_key="my-api-key"
            )
    """
    model: str = "embedding-english-v1"
    """Model name to use."""
    api_key: Optional[str] = None
    """LLMRails API key."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Call out to Cohere's embedding endpoint.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        api_key = self.api_key or os.environ.get("LLM_RAILS_API_KEY")
        if api_key is None:
            logging.warning("Can't find LLMRails credentials in environment.")
            raise ValueError("LLM_RAILS_API_KEY is not set")
raise ValueError("LLM_RAILS_API_KEY is not set")
        response = requests.post(
            "https://api.llmrails.com/v1/embeddings",
            headers={"X-API-KEY": api_key},
            json={"input": texts, "model": self.model},
            timeout=60,
        )
        return [item["embedding"] for item in response.json()["data"]]
[docs]    def embed_query(self, text: str) -> List[float]:
        """Call out to Cohere's embedding endpoint.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        return self.embed_documents([text])[0]
Source code for langchain_community.embeddings.embaas
from typing import Any, Dict, List, Mapping, Optional
import requests
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_core.utils import get_from_dict_or_env
from requests.adapters import HTTPAdapter, Retry
from typing_extensions import NotRequired, TypedDict
# Currently supported maximum batch size for embedding requests
MAX_BATCH_SIZE = 256
EMBAAS_API_URL = "https://api.embaas.io/v1/embeddings/"
[docs]class EmbaasEmbeddingsPayload(TypedDict):
    """Payload for the Embaas embeddings API."""
    model: str
    texts: List[str]
    instruction: NotRequired[str]
[docs]class EmbaasEmbeddings(BaseModel, Embeddings):
    """Embaas's embedding service.
    To use, you should have the
    environment variable ``EMBAAS_API_KEY`` set with your API key, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            # Initialise with default model and instruction
            from langchain_community.embeddings import EmbaasEmbeddings
            emb = EmbaasEmbeddings()
            # Initialise with custom model and instruction
            from langchain_community.embeddings import EmbaasEmbeddings
            emb_model = "instructor-large"
            emb_inst = "Represent the Wikipedia document for retrieval"
            emb = EmbaasEmbeddings(
                model=emb_model,
                instruction=emb_inst
            )
    """
    model: str = "e5-large-v2"
    """The model used for embeddings."""
    instruction: Optional[str] = None
    """Instruction used for domain-specific embeddings."""
instruction: Optional[str] = None
    """Instruction used for domain-specific embeddings."""
    api_url: str = EMBAAS_API_URL
    """The URL for the embaas embeddings API."""
    embaas_api_key: Optional[str] = None
    """max number of retries for requests"""
    max_retries: Optional[int] = 3
    """request timeout in seconds"""
    timeout: Optional[int] = 30
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        embaas_api_key = get_from_dict_or_env(
            values, "embaas_api_key", "EMBAAS_API_KEY"
        )
        values["embaas_api_key"] = embaas_api_key
        return values
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        """Get the identifying params."""
        return {"model": self.model, "instruction": self.instruction}
    def _generate_payload(self, texts: List[str]) -> EmbaasEmbeddingsPayload:
        """Generates payload for the API request."""
        payload = EmbaasEmbeddingsPayload(texts=texts, model=self.model)
        if self.instruction:
            payload["instruction"] = self.instruction
        return payload
    def _handle_request(self, payload: EmbaasEmbeddingsPayload) -> List[List[float]]:
        """Sends a request to the Embaas API and handles the response."""
        headers = {
            "Authorization": f"Bearer {self.embaas_api_key}",
            "Content-Type": "application/json",
        }
"Content-Type": "application/json",
        }
        session = requests.Session()
        retries = Retry(
            total=self.max_retries,
            backoff_factor=0.5,
            allowed_methods=["POST"],
            raise_on_status=True,
        )
        session.mount("http://", HTTPAdapter(max_retries=retries))
        session.mount("https://", HTTPAdapter(max_retries=retries))
        response = session.post(
            self.api_url,
            headers=headers,
            json=payload,
            timeout=self.timeout,
        )
        parsed_response = response.json()
        embeddings = [item["embedding"] for item in parsed_response["data"]]
        return embeddings
    def _generate_embeddings(self, texts: List[str]) -> List[List[float]]:
        """Generate embeddings using the Embaas API."""
        payload = self._generate_payload(texts)
        try:
            return self._handle_request(payload)
        except requests.exceptions.RequestException as e:
            if e.response is None or not e.response.text:
                raise ValueError(f"Error raised by embaas embeddings API: {e}")
            parsed_response = e.response.json()
            if "message" in parsed_response:
                raise ValueError(
                    "Validation Error raised by embaas embeddings API:"
                    f"{parsed_response['message']}"
                )
            raise
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Get embeddings for a list of texts.
        Args:
            texts: The list of texts to get embeddings for.
        Returns:
            List of embeddings, one for each text.
        """
        batches = [
List of embeddings, one for each text.
        """
        batches = [
            texts[i : i + MAX_BATCH_SIZE] for i in range(0, len(texts), MAX_BATCH_SIZE)
        ]
        embeddings = [self._generate_embeddings(batch) for batch in batches]
        # flatten the list of lists into a single list
        return [embedding for batch in embeddings for embedding in batch]
[docs]    def embed_query(self, text: str) -> List[float]:
        """Get embeddings for a single text.
        Args:
            text: The text to get embeddings for.
        Returns:
            List of embeddings.
        """
        return self.embed_documents([text])[0]
Source code for langchain_community.embeddings.cloudflare_workersai
from typing import Any, Dict, List
import requests
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra
DEFAULT_MODEL_NAME = "@cf/baai/bge-base-en-v1.5"
[docs]class CloudflareWorkersAIEmbeddings(BaseModel, Embeddings):
    """Cloudflare Workers AI embedding model.
    To use, you need to provide an API token and
    account ID to access Cloudflare Workers AI.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import CloudflareWorkersAIEmbeddings
            account_id = "my_account_id"
            api_token = "my_secret_api_token"
            model_name =  "@cf/baai/bge-small-en-v1.5"
            cf = CloudflareWorkersAIEmbeddings(
                account_id=account_id,
                api_token=api_token,
                model_name=model_name
            )
    """
    api_base_url: str = "https://api.cloudflare.com/client/v4/accounts"
    account_id: str
    api_token: str
    model_name: str = DEFAULT_MODEL_NAME
    batch_size: int = 50
    strip_new_lines: bool = True
    headers: Dict[str, str] = {"Authorization": "Bearer "}
    def __init__(self, **kwargs: Any):
        """Initialize the Cloudflare Workers AI client."""
        super().__init__(**kwargs)
        self.headers = {"Authorization": f"Bearer {self.api_token}"}
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
"""Compute doc embeddings using Cloudflare Workers AI.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        if self.strip_new_lines:
            texts = [text.replace("\n", " ") for text in texts]
        batches = [
            texts[i : i + self.batch_size]
            for i in range(0, len(texts), self.batch_size)
        ]
        embeddings = []
        for batch in batches:
            response = requests.post(
                f"{self.api_base_url}/{self.account_id}/ai/run/{self.model_name}",
                headers=self.headers,
                json={"text": batch},
            )
            embeddings.extend(response.json()["result"]["data"])
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using Cloudflare Workers AI.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        text = text.replace("\n", " ") if self.strip_new_lines else text
        response = requests.post(
            f"{self.api_base_url}/{self.account_id}/ai/run/{self.model_name}",
            headers=self.headers,
            json={"text": [text]},
        )
        return response.json()["result"]["data"][0]
Source code for langchain_community.embeddings.sagemaker_endpoint
from typing import Any, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_community.llms.sagemaker_endpoint import ContentHandlerBase
[docs]class EmbeddingsContentHandler(ContentHandlerBase[List[str], List[List[float]]]):
    """Content handler for LLM class."""
[docs]class SagemakerEndpointEmbeddings(BaseModel, Embeddings):
    """Custom Sagemaker Inference Endpoints.
    To use, you must supply the endpoint name from your deployed
    Sagemaker model & the region where it is deployed.
    To authenticate, the AWS client uses the following methods to
    automatically load credentials:
    https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    If a specific credential profile should be used, you must pass
    the name of the profile from the ~/.aws/credentials file that is to be used.
    Make sure the credentials / roles used have the required policies to
    access the Sagemaker endpoint.
    See: https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html
    """
    """
    Example:
        .. code-block:: python
            from langchain_community.embeddings import SagemakerEndpointEmbeddings
            endpoint_name = (
                "my-endpoint-name"
            )
            region_name = (
                "us-west-2"
            )
            credentials_profile_name = (
                "default"
            )
            se = SagemakerEndpointEmbeddings(
                endpoint_name=endpoint_name,
                region_name=region_name,
                credentials_profile_name=credentials_profile_name
            )
            #Use with boto3 client
credentials_profile_name=credentials_profile_name
            )
            #Use with boto3 client
            client = boto3.client(
                        "sagemaker-runtime",
                        region_name=region_name
                    )
            se = SagemakerEndpointEmbeddings(
                endpoint_name=endpoint_name,
                client=client
            )
    """
    client: Any = None
    endpoint_name: str = ""
    """The name of the endpoint from the deployed Sagemaker model.
    Must be unique within an AWS Region."""
    region_name: str = ""
    """The aws region where the Sagemaker model is deployed, eg. `us-west-2`."""
    credentials_profile_name: Optional[str] = None
    """The name of the profile in the ~/.aws/credentials or ~/.aws/config files, which
    has either access keys or role information specified.
    If not specified, the default credential profile or, if on an EC2 instance,
    credentials from IMDS will be used.
    See: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
    """
    content_handler: EmbeddingsContentHandler
    """The content handler class that provides an input and
    output transform functions to handle formats between LLM
    and the endpoint.
    """
    """
     Example:
        .. code-block:: python
        from langchain_community.embeddings.sagemaker_endpoint import EmbeddingsContentHandler
        class ContentHandler(EmbeddingsContentHandler):
                content_type = "application/json"
                accepts = "application/json"
                def transform_input(self, prompts: List[str], model_kwargs: Dict) -> bytes:
                    input_str = json.dumps({prompts: prompts, **model_kwargs})
                    return input_str.encode('utf-8')
return input_str.encode('utf-8')
                def transform_output(self, output: bytes) -> List[List[float]]:
                    response_json = json.loads(output.read().decode("utf-8"))
                    return response_json["vectors"]
    """  # noqa: E501
    model_kwargs: Optional[Dict] = None
    """Keyword arguments to pass to the model."""
    endpoint_kwargs: Optional[Dict] = None
    """Optional attributes passed to the invoke_endpoint
    function. See `boto3`_. docs for more info.
    .. _boto3: <https://boto3.amazonaws.com/v1/documentation/api/latest/index.html>
    """
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
        arbitrary_types_allowed = True
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Dont do anything if client provided externally"""
        if values.get("client") is not None:
            return values
        """Validate that AWS credentials to and python package exists in environment."""
        try:
            import boto3
            try:
                if values["credentials_profile_name"] is not None:
                    session = boto3.Session(
                        profile_name=values["credentials_profile_name"]
                    )
                else:
                    # use default credentials
                    session = boto3.Session()
                values["client"] = session.client(
                    "sagemaker-runtime", region_name=values["region_name"]
                )
            except Exception as e:
                raise ValueError(
                    "Could not load credentials to authenticate with AWS client. "
                    "Please check that credentials in the specified "
                    "profile name are valid."
                ) from e
        except ImportError:
            raise ImportError(
) from e
        except ImportError:
            raise ImportError(
                "Could not import boto3 python package. "
                "Please install it with `pip install boto3`."
            )
        return values
    def _embedding_func(self, texts: List[str]) -> List[List[float]]:
        """Call out to SageMaker Inference embedding endpoint."""
        # replace newlines, which can negatively affect performance.
        texts = list(map(lambda x: x.replace("\n", " "), texts))
        _model_kwargs = self.model_kwargs or {}
        _endpoint_kwargs = self.endpoint_kwargs or {}
        body = self.content_handler.transform_input(texts, _model_kwargs)
        content_type = self.content_handler.content_type
        accepts = self.content_handler.accepts
        # send request
        try:
            response = self.client.invoke_endpoint(
                EndpointName=self.endpoint_name,
                Body=body,
                ContentType=content_type,
                Accept=accepts,
                **_endpoint_kwargs,
            )
        except Exception as e:
            raise ValueError(f"Error raised by inference endpoint: {e}")
        return self.content_handler.transform_output(response["Body"])
[docs]    def embed_documents(
        self, texts: List[str], chunk_size: int = 64
    ) -> List[List[float]]:
        """Compute doc embeddings using a SageMaker Inference Endpoint.
        Args:
            texts: The list of texts to embed.
            chunk_size: The chunk size defines how many input texts will
                be grouped together as request. If None, will use the
                chunk size specified by the class.
        Returns:
            List of embeddings, one for each text.
        """
        results = []
List of embeddings, one for each text.
        """
        results = []
        _chunk_size = len(texts) if chunk_size > len(texts) else chunk_size
        for i in range(0, len(texts), _chunk_size):
            response = self._embedding_func(texts[i : i + _chunk_size])
            results.extend(response)
        return results
[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using a SageMaker inference endpoint.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        return self._embedding_func([text])[0]
Source code for langchain_community.embeddings.modelscope_hub
from typing import Any, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra
[docs]class ModelScopeEmbeddings(BaseModel, Embeddings):
    """ModelScopeHub embedding models.
    To use, you should have the ``modelscope`` python package installed.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import ModelScopeEmbeddings
            model_id = "damo/nlp_corom_sentence-embedding_english-base"
            embed = ModelScopeEmbeddings(model_id=model_id, model_revision="v1.0.0")
    """
    embed: Any
    model_id: str = "damo/nlp_corom_sentence-embedding_english-base"
    """Model name to use."""
    model_revision: Optional[str] = None
    def __init__(self, **kwargs: Any):
        """Initialize the modelscope"""
        super().__init__(**kwargs)
        try:
            from modelscope.pipelines import pipeline
            from modelscope.utils.constant import Tasks
        except ImportError as e:
            raise ImportError(
                "Could not import some python packages."
                "Please install it with `pip install modelscope`."
            ) from e
        self.embed = pipeline(
            Tasks.sentence_embedding,
            model=self.model_id,
            model_revision=self.model_revision,
        )
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Compute doc embeddings using a modelscope embedding model.
        Args:
            texts: The list of texts to embed.
        Returns:
Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        texts = list(map(lambda x: x.replace("\n", " "), texts))
        inputs = {"source_sentence": texts}
        embeddings = self.embed(input=inputs)["text_embedding"]
        return embeddings.tolist()
[docs]    def embed_query(self, text: str) -> List[float]:
        """Compute query embeddings using a modelscope embedding model.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        text = text.replace("\n", " ")
        inputs = {"source_sentence": [text]}
        embedding = self.embed(input=inputs)["text_embedding"][0]
        return embedding.tolist()