viruPVS/langchain-docs huggingface dataset:
# and text batches for the rest of the texts.
        """
        from google.api_core.exceptions import InvalidArgument
        batches = VertexAIEmbeddings._prepare_batches(
            texts, self.instance["batch_size"]
        )
        # If batch size if less or equal to one that went through before,
        # then keep batches as they are.
        if len(batches[0]) <= self.instance["min_good_batch_size"]:
            return [], batches
        with self.instance["lock"]:
            # If largest possible batch size was validated
            # while waiting for the lock, then check for rebuilding
            # our batches, and return.
            if self.instance["batch_size_validated"]:
                if len(batches[0]) <= self.instance["batch_size"]:
                    return [], batches
                else:
                    return [], VertexAIEmbeddings._prepare_batches(
                        texts, self.instance["batch_size"]
                    )
            # Figure out largest possible batch size by trying to push
            # batches and lowering their size in half after every failure.
            first_batch = batches[0]
            first_result = []
            had_failure = False
            while True:
                try:
                    first_result = self._get_embeddings_with_retry(
                        first_batch, embeddings_type
                    )
                    break
                except InvalidArgument:
                    had_failure = True
                    first_batch_len = len(first_batch)
                    if first_batch_len == self.instance["min_batch_size"]:
                        raise
                    first_batch_len = max(
                        self.instance["min_batch_size"], int(first_batch_len / 2)
                    )
                    first_batch = first_batch[:first_batch_len]
            first_batch_len = len(first_batch)
            self.instance["min_good_batch_size"] = max(
                self.instance["min_good_batch_size"], first_batch_len
self.instance["min_good_batch_size"], first_batch_len
            )
            # If had a failure and recovered
            # or went through with the max size, then it's a legit batch size.
            if had_failure or first_batch_len == self.instance["max_batch_size"]:
                self.instance["batch_size"] = first_batch_len
                self.instance["batch_size_validated"] = True
                # If batch size was updated,
                # rebuild batches with the new batch size
                # (texts that went through are excluded here).
                if first_batch_len != self.instance["max_batch_size"]:
                    batches = VertexAIEmbeddings._prepare_batches(
                        texts[first_batch_len:], self.instance["batch_size"]
                    )
            else:
                # Still figuring out max batch size.
                batches = batches[1:]
        # Returning embeddings of the first text batch that went through,
        # and text batches for the rest of texts.
        return first_result, batches
[docs]    def embed(
        self,
        texts: List[str],
        batch_size: int = 0,
        embeddings_task_type: Optional[
            Literal[
                "RETRIEVAL_QUERY",
                "RETRIEVAL_DOCUMENT",
                "SEMANTIC_SIMILARITY",
                "CLASSIFICATION",
                "CLUSTERING",
            ]
        ] = None,
    ) -> List[List[float]]:
        """Embed a list of strings.
        Args:
            texts: List[str] The list of strings to embed.
            batch_size: [int] The batch size of embeddings to send to the model.
                If zero, then the largest batch size will be detected dynamically
                at the first request, starting from 250, down to 5.
at the first request, starting from 250, down to 5.
            embeddings_task_type: [str] optional embeddings task type,
                one of the following
                    RETRIEVAL_QUERY	- Text is a query
                                      in a search/retrieval setting.
                    RETRIEVAL_DOCUMENT - Text is a document
                                         in a search/retrieval setting.
                    SEMANTIC_SIMILARITY - Embeddings will be used
                                          for Semantic Textual Similarity (STS).
                    CLASSIFICATION - Embeddings will be used for classification.
                    CLUSTERING - Embeddings will be used for clustering.
        Returns:
            List of embeddings, one for each text.
        """
        if len(texts) == 0:
            return []
        embeddings: List[List[float]] = []
        first_batch_result: List[List[float]] = []
        if batch_size > 0:
            # Fixed batch size.
            batches = VertexAIEmbeddings._prepare_batches(texts, batch_size)
        else:
            # Dynamic batch size, starting from 250 at the first call.
            first_batch_result, batches = self._prepare_and_validate_batches(
                texts, embeddings_task_type
            )
        # First batch result may have some embeddings already.
        # In such case, batches have texts that were not processed yet.
        embeddings.extend(first_batch_result)
        tasks = []
        for batch in batches:
            tasks.append(
                self.instance["task_executor"].submit(
                    self._get_embeddings_with_retry,
                    texts=batch,
                    embeddings_type=embeddings_task_type,
                )
            )
        if len(tasks) > 0:
            wait(tasks)
        for t in tasks:
            embeddings.extend(t.result())
        return embeddings
for t in tasks:
            embeddings.extend(t.result())
        return embeddings
[docs]    def embed_documents(
        self, texts: List[str], batch_size: int = 0
    ) -> List[List[float]]:
        """Embed a list of documents.
        Args:
            texts: List[str] The list of texts to embed.
            batch_size: [int] The batch size of embeddings to send to the model.
                If zero, then the largest batch size will be detected dynamically
                at the first request, starting from 250, down to 5.
        Returns:
            List of embeddings, one for each text.
        """
        return self.embed(texts, batch_size, "RETRIEVAL_DOCUMENT")
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a text.
        Args:
            text: The text to embed.
        Returns:
            Embedding for the text.
        """
        embeddings = self.embed([text], 1, "RETRIEVAL_QUERY")
        return embeddings[0]
Source code for langchain_community.embeddings.spacy_embeddings
import importlib.util
from typing import Any, Dict, List
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
[docs]class SpacyEmbeddings(BaseModel, Embeddings):
    """Embeddings by SpaCy models.
    It only supports the 'en_core_web_sm' model.
    Attributes:
        nlp (Any): The Spacy model loaded into memory.
    Methods:
        embed_documents(texts: List[str]) -> List[List[float]]:
            Generates embeddings for a list of documents.
        embed_query(text: str) -> List[float]:
            Generates an embedding for a single piece of text.
    """
    nlp: Any  # The Spacy model loaded into memory
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid  # Forbid extra attributes during model initialization
    @root_validator(pre=True)
    def validate_environment(cls, values: Dict) -> Dict:
        """
        Validates that the Spacy package and the 'en_core_web_sm' model are installed.
        Args:
            values (Dict): The values provided to the class constructor.
        Returns:
            The validated values.
        Raises:
            ValueError: If the Spacy package or the 'en_core_web_sm'
            model are not installed.
        """
        # Check if the Spacy package is installed
        if importlib.util.find_spec("spacy") is None:
            raise ValueError(
                "Spacy package not found. "
                "Please install it with `pip install spacy`."
            )
        try:
            # Try to load the 'en_core_web_sm' Spacy model
try:
            # Try to load the 'en_core_web_sm' Spacy model
            import spacy
            values["nlp"] = spacy.load("en_core_web_sm")
        except OSError:
            # If the model is not found, raise a ValueError
            raise ValueError(
                "Spacy model 'en_core_web_sm' not found. "
                "Please install it with"
                " `python -m spacy download en_core_web_sm`."
            )
        return values  # Return the validated values
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """
        Generates embeddings for a list of documents.
        Args:
            texts (List[str]): The documents to generate embeddings for.
        Returns:
            A list of embeddings, one for each document.
        """
        return [self.nlp(text).vector.tolist() for text in texts]
[docs]    def embed_query(self, text: str) -> List[float]:
        """
        Generates an embedding for a single piece of text.
        Args:
            text (str): The text to generate an embedding for.
        Returns:
            The embedding for the text.
        """
        return self.nlp(text).vector.tolist()
[docs]    async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
        """
        Asynchronously generates embeddings for a list of documents.
        This method is not implemented and raises a NotImplementedError.
        Args:
            texts (List[str]): The documents to generate embeddings for.
        Raises:
            NotImplementedError: This method is not implemented.
        """
        raise NotImplementedError("Asynchronous embedding generation is not supported.")
"""
        raise NotImplementedError("Asynchronous embedding generation is not supported.")
[docs]    async def aembed_query(self, text: str) -> List[float]:
        """
        Asynchronously generates an embedding for a single piece of text.
        This method is not implemented and raises a NotImplementedError.
        Args:
            text (str): The text to generate an embedding for.
        Raises:
            NotImplementedError: This method is not implemented.
        """
        raise NotImplementedError("Asynchronous embedding generation is not supported.")
Source code for langchain_community.embeddings.baidu_qianfan_endpoint
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, root_validator
from langchain_core.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class QianfanEmbeddingsEndpoint(BaseModel, Embeddings):
    """`Baidu Qianfan Embeddings` embedding models."""
    qianfan_ak: Optional[str] = None
    """Qianfan application apikey"""
    qianfan_sk: Optional[str] = None
    """Qianfan application secretkey"""
    chunk_size: int = 16
    """Chunk size when multiple texts are input"""
    model: str = "Embedding-V1"
    """Model name
    you could get from https://cloud.baidu.com/doc/WENXINWORKSHOP/s/Nlks5zkzu
    
    for now, we support Embedding-V1 and 
    - Embedding-V1 （默认模型）
    - bge-large-en
    - bge-large-zh
    
    preset models are mapping to an endpoint.
    `model` will be ignored if `endpoint` is set
    """
    endpoint: str = ""
    """Endpoint of the Qianfan Embedding, required if custom model used."""
    client: Any
    """Qianfan client"""
    max_retries: int = 5
    """Max reties times"""
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """
        Validate whether qianfan_ak and qianfan_sk in the environment variables or
        configuration file are available or not.
configuration file are available or not.
        init qianfan embedding client with `ak`, `sk`, `model`, `endpoint`
        Args:
            values: a dictionary containing configuration information, must include the
            fields of qianfan_ak and qianfan_sk
        Returns:
            a dictionary containing configuration information. If qianfan_ak and
            qianfan_sk are not provided in the environment variables or configuration
            file,the original values will be returned; otherwise, values containing
            qianfan_ak and qianfan_sk will be returned.
        Raises:
            ValueError: qianfan package not found, please install it with `pip install
            qianfan`
        """
        values["qianfan_ak"] = get_from_dict_or_env(
            values,
            "qianfan_ak",
            "QIANFAN_AK",
        )
        values["qianfan_sk"] = get_from_dict_or_env(
            values,
            "qianfan_sk",
            "QIANFAN_SK",
        )
        try:
            import qianfan
            params = {
                "ak": values["qianfan_ak"],
                "sk": values["qianfan_sk"],
                "model": values["model"],
            }
            if values["endpoint"] is not None and values["endpoint"] != "":
                params["endpoint"] = values["endpoint"]
            values["client"] = qianfan.Embedding(**params)
        except ImportError:
            raise ImportError(
                "qianfan package not found, please install it with "
                "`pip install qianfan`"
            )
        return values
[docs]    def embed_query(self, text: str) -> List[float]:
        resp = self.embed_documents([text])
resp = self.embed_documents([text])
        return resp[0]
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """
        Embeds a list of text documents using the AutoVOT algorithm.
        Args:
            texts (List[str]): A list of text documents to embed.
        Returns:
            List[List[float]]: A list of embeddings for each document in the input list.
                            Each embedding is represented as a list of float values.
        """
        text_in_chunks = [
            texts[i : i + self.chunk_size]
            for i in range(0, len(texts), self.chunk_size)
        ]
        lst = []
        for chunk in text_in_chunks:
            resp = self.client.do(texts=chunk)
            lst.extend([res["embedding"] for res in resp["data"]])
        return lst
[docs]    async def aembed_query(self, text: str) -> List[float]:
        embeddings = await self.aembed_documents([text])
        return embeddings[0]
[docs]    async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
        text_in_chunks = [
            texts[i : i + self.chunk_size]
            for i in range(0, len(texts), self.chunk_size)
        ]
        lst = []
        for chunk in text_in_chunks:
            resp = await self.client.ado(texts=chunk)
            for res in resp["data"]:
                lst.extend([res["embedding"]])
        return lst
Source code for langchain_community.embeddings.xinference
"""Wrapper around Xinference embedding models."""
from typing import Any, List, Optional
from langchain_core.embeddings import Embeddings
[docs]class XinferenceEmbeddings(Embeddings):
    """Xinference embedding models.
    To use, you should have the xinference library installed:
    .. code-block:: bash
        pip install xinference
    Check out: https://github.com/xorbitsai/inference
    To run, you need to start a Xinference supervisor on one server and Xinference workers on the other servers.
    Example:
        To start a local instance of Xinference, run
        .. code-block:: bash
           $ xinference
        You can also deploy Xinference in a distributed cluster. Here are the steps:
        Starting the supervisor:
        .. code-block:: bash
           $ xinference-supervisor
        Starting the worker:
        .. code-block:: bash
           $ xinference-worker
    Then, launch a model using command line interface (CLI).
    Example:
    .. code-block:: bash
       $ xinference launch -n orca -s 3 -q q4_0
    It will return a model UID. Then you can use Xinference Embedding with LangChain.
    Example:
    .. code-block:: python
        from langchain_community.embeddings import XinferenceEmbeddings
        xinference = XinferenceEmbeddings(
            server_url="http://0.0.0.0:9997",
            model_uid = {model_uid} # replace model_uid with the model UID return from launching the model
        )
    """  # noqa: E501
    client: Any
    server_url: Optional[str]
    """URL of the xinference server"""
server_url: Optional[str]
    """URL of the xinference server"""
    model_uid: Optional[str]
    """UID of the launched model"""
[docs]    def __init__(
        self, server_url: Optional[str] = None, model_uid: Optional[str] = None
    ):
        try:
            from xinference.client import RESTfulClient
        except ImportError as e:
            raise ImportError(
                "Could not import RESTfulClient from xinference. Please install it"
                " with `pip install xinference`."
            ) from e
        super().__init__()
        if server_url is None:
            raise ValueError("Please provide server URL")
        if model_uid is None:
            raise ValueError("Please provide the model UID")
        self.server_url = server_url
        self.model_uid = model_uid
        self.client = RESTfulClient(server_url)
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed a list of documents using Xinference.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        model = self.client.get_model(self.model_uid)
        embeddings = [
            model.create_embedding(text)["data"][0]["embedding"] for text in texts
        ]
        return [list(map(float, e)) for e in embeddings]
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a query of documents using Xinference.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        model = self.client.get_model(self.model_uid)
        embedding_res = model.create_embedding(text)
Source code for langchain_community.embeddings.minimax
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, List, Optional
import requests
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_core.utils import get_from_dict_or_env
from tenacity import (
    before_sleep_log,
    retry,
    stop_after_attempt,
    wait_exponential,
)
logger = logging.getLogger(__name__)
def _create_retry_decorator() -> Callable[[Any], Any]:
    """Returns a tenacity retry decorator."""
    multiplier = 1
    min_seconds = 1
    max_seconds = 4
    max_retries = 6
    return retry(
        reraise=True,
        stop=stop_after_attempt(max_retries),
        wait=wait_exponential(multiplier=multiplier, min=min_seconds, max=max_seconds),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
[docs]def embed_with_retry(embeddings: MiniMaxEmbeddings, *args: Any, **kwargs: Any) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator()
    @retry_decorator
    def _embed_with_retry(*args: Any, **kwargs: Any) -> Any:
        return embeddings.embed(*args, **kwargs)
    return _embed_with_retry(*args, **kwargs)
[docs]class MiniMaxEmbeddings(BaseModel, Embeddings):
    """MiniMax's embedding service.
    To use, you should have the environment variable ``MINIMAX_GROUP_ID`` and
    ``MINIMAX_API_KEY`` set with your API token, or pass it as a named parameter to
    the constructor.
the constructor.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import MiniMaxEmbeddings
            embeddings = MiniMaxEmbeddings()
            query_text = "This is a test query."
            query_result = embeddings.embed_query(query_text)
            document_text = "This is a test document."
            document_result = embeddings.embed_documents([document_text])
    """
    endpoint_url: str = "https://api.minimax.chat/v1/embeddings"
    """Endpoint URL to use."""
    model: str = "embo-01"
    """Embeddings model name to use."""
    embed_type_db: str = "db"
    """For embed_documents"""
    embed_type_query: str = "query"
    """For embed_query"""
    minimax_group_id: Optional[str] = None
    """Group ID for MiniMax API."""
    minimax_api_key: Optional[str] = None
    """API Key for MiniMax API."""
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that group id and api key exists in environment."""
        minimax_group_id = get_from_dict_or_env(
            values, "minimax_group_id", "MINIMAX_GROUP_ID"
        )
        minimax_api_key = get_from_dict_or_env(
            values, "minimax_api_key", "MINIMAX_API_KEY"
        )
        values["minimax_group_id"] = minimax_group_id
        values["minimax_api_key"] = minimax_api_key
        return values
[docs]    def embed(
        self,
        texts: List[str],
        embed_type: str,
self,
        texts: List[str],
        embed_type: str,
    ) -> List[List[float]]:
        payload = {
            "model": self.model,
            "type": embed_type,
            "texts": texts,
        }
        # HTTP headers for authorization
        headers = {
            "Authorization": f"Bearer {self.minimax_api_key}",
            "Content-Type": "application/json",
        }
        params = {
            "GroupId": self.minimax_group_id,
        }
        # send request
        response = requests.post(
            self.endpoint_url, params=params, headers=headers, json=payload
        )
        parsed_response = response.json()
        # check for errors
        if parsed_response["base_resp"]["status_code"] != 0:
            raise ValueError(
                f"MiniMax API returned an error: {parsed_response['base_resp']}"
            )
        embeddings = parsed_response["vectors"]
        return embeddings
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Embed documents using a MiniMax embedding endpoint.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        embeddings = embed_with_retry(self, texts=texts, embed_type=self.embed_type_db)
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed a query using a MiniMax embedding endpoint.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        embeddings = embed_with_retry(
            self, texts=[text], embed_type=self.embed_type_query
        )
        return embeddings[0]