viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.embeddings.fake
import hashlib
from typing import List
import numpy as np
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel
[docs]class FakeEmbeddings(Embeddings, BaseModel):
    """Fake embedding model."""
    size: int
    """The size of the embedding vector."""
    def _get_embedding(self) -> List[float]:
        return list(np.random.normal(size=self.size))
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        return [self._get_embedding() for _ in texts]
[docs]    def embed_query(self, text: str) -> List[float]:
        return self._get_embedding()
[docs]class DeterministicFakeEmbedding(Embeddings, BaseModel):
    """
    Fake embedding model that always returns
    the same embedding vector for the same text.
    """
    size: int
    """The size of the embedding vector."""
    def _get_embedding(self, seed: int) -> List[float]:
        # set the seed for the random generator
        np.random.seed(seed)
        return list(np.random.normal(size=self.size))
    def _get_seed(self, text: str) -> int:
        """
        Get a seed for the random generator, using the hash of the text.
        """
        return int(hashlib.sha256(text.encode("utf-8")).hexdigest(), 16) % 10**8
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        return [self._get_embedding(seed=self._get_seed(_)) for _ in texts]
[docs]    def embed_query(self, text: str) -> List[float]:
        return self._get_embedding(seed=self._get_seed(text))
Source code for langchain_community.embeddings.javelin_ai_gateway
from __future__ import annotations
from typing import Any, Iterator, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel
def _chunk(texts: List[str], size: int) -> Iterator[List[str]]:
    for i in range(0, len(texts), size):
        yield texts[i : i + size]
[docs]class JavelinAIGatewayEmbeddings(Embeddings, BaseModel):
    """
    Wrapper around embeddings LLMs in the Javelin AI Gateway.
    To use, you should have the ``javelin_sdk`` python package installed.
    For more information, see https://docs.getjavelin.io
    Example:
        .. code-block:: python
            from langchain_community.embeddings import JavelinAIGatewayEmbeddings
            embeddings = JavelinAIGatewayEmbeddings(
                gateway_uri="<javelin-ai-gateway-uri>",
                route="<your-javelin-gateway-embeddings-route>"
            )
    """
    client: Any
    """javelin client."""
    route: str
    """The route to use for the Javelin AI Gateway API."""
    gateway_uri: Optional[str] = None
    """The URI for the Javelin AI Gateway API."""
    javelin_api_key: Optional[str] = None
    """The API key for the Javelin AI Gateway API."""
    def __init__(self, **kwargs: Any):
        try:
            from javelin_sdk import (
                JavelinClient,
                UnauthorizedError,
            )
        except ImportError:
            raise ImportError(
                "Could not import javelin_sdk python package. "
raise ImportError(
                "Could not import javelin_sdk python package. "
                "Please install it with `pip install javelin_sdk`."
            )
        super().__init__(**kwargs)
        if self.gateway_uri:
            try:
                self.client = JavelinClient(
                    base_url=self.gateway_uri, api_key=self.javelin_api_key
                )
            except UnauthorizedError as e:
                raise ValueError("Javelin: Incorrect API Key.") from e
    def _query(self, texts: List[str]) -> List[List[float]]:
        embeddings = []
        for txt in _chunk(texts, 20):
            try:
                resp = self.client.query_route(self.route, query_body={"input": txt})
                resp_dict = resp.dict()
                embeddings_chunk = resp_dict.get("llm_response", {}).get("data", [])
                for item in embeddings_chunk:
                    if "embedding" in item:
                        embeddings.append(item["embedding"])
            except ValueError as e:
                print("Failed to query route: " + str(e))
        return embeddings
    async def _aquery(self, texts: List[str]) -> List[List[float]]:
        embeddings = []
        for txt in _chunk(texts, 20):
            try:
                resp = await self.client.aquery_route(
                    self.route, query_body={"input": txt}
                )
                resp_dict = resp.dict()
                embeddings_chunk = resp_dict.get("llm_response", {}).get("data", [])
                for item in embeddings_chunk:
                    if "embedding" in item:
                        embeddings.append(item["embedding"])
            except ValueError as e:
                print("Failed to query route: " + str(e))
        return embeddings
print("Failed to query route: " + str(e))
        return embeddings
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        return self._query(texts)
[docs]    def embed_query(self, text: str) -> List[float]:
        return self._query([text])[0]
[docs]    async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
        return await self._aquery(texts)
[docs]    async def aembed_query(self, text: str) -> List[float]:
        result = await self._aquery([text])
        return result[0]
Source code for langchain_community.embeddings.google_palm
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, root_validator
from langchain_core.utils import get_from_dict_or_env
from tenacity import (
    before_sleep_log,
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential,
)
logger = logging.getLogger(__name__)
def _create_retry_decorator() -> Callable[[Any], Any]:
    """Returns a tenacity retry decorator, preconfigured to handle PaLM exceptions"""
    import google.api_core.exceptions
    multiplier = 2
    min_seconds = 1
    max_seconds = 60
    max_retries = 10
    return retry(
        reraise=True,
        stop=stop_after_attempt(max_retries),
        wait=wait_exponential(multiplier=multiplier, min=min_seconds, max=max_seconds),
        retry=(
            retry_if_exception_type(google.api_core.exceptions.ResourceExhausted)
            | retry_if_exception_type(google.api_core.exceptions.ServiceUnavailable)
            | retry_if_exception_type(google.api_core.exceptions.GoogleAPIError)
        ),
        before_sleep=before_sleep_log(logger, logging.WARNING),
    )
[docs]def embed_with_retry(
    embeddings: GooglePalmEmbeddings, *args: Any, **kwargs: Any
) -> Any:
    """Use tenacity to retry the completion call."""
    retry_decorator = _create_retry_decorator()
    @retry_decorator
    def _embed_with_retry(*args: Any, **kwargs: Any) -> Any:
        return embeddings.client.generate_embeddings(*args, **kwargs)
return embeddings.client.generate_embeddings(*args, **kwargs)
    return _embed_with_retry(*args, **kwargs)
[docs]class GooglePalmEmbeddings(BaseModel, Embeddings):
    """Google's PaLM Embeddings APIs."""
    client: Any
    google_api_key: Optional[str]
    model_name: str = "models/embedding-gecko-001"
    """Model name to use."""
    show_progress_bar: bool = False
    """Whether to show a tqdm progress bar. Must have `tqdm` installed."""
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate api key, python package exists."""
        google_api_key = get_from_dict_or_env(
            values, "google_api_key", "GOOGLE_API_KEY"
        )
        try:
            import google.generativeai as genai
            genai.configure(api_key=google_api_key)
        except ImportError:
            raise ImportError("Could not import google.generativeai python package.")
        values["client"] = genai
        return values
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        if self.show_progress_bar:
            try:
                from tqdm import tqdm
                iter_ = tqdm(texts, desc="GooglePalmEmbeddings")
            except ImportError:
                logger.warning(
                    "Unable to show progress bar because tqdm could not be imported. "
                    "Please install with `pip install tqdm`."
                )
                iter_ = texts
        else:
            iter_ = texts
        return [self.embed_query(text) for text in iter_]
[docs]    def embed_query(self, text: str) -> List[float]:
        """Embed query text."""
Source code for langchain_community.embeddings.mlflow
from __future__ import annotations
from typing import Any, Iterator, List
from urllib.parse import urlparse
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, PrivateAttr
def _chunk(texts: List[str], size: int) -> Iterator[List[str]]:
    for i in range(0, len(texts), size):
        yield texts[i : i + size]
[docs]class MlflowEmbeddings(Embeddings, BaseModel):
    """Wrapper around embeddings LLMs in MLflow.
    To use, you should have the `mlflow[genai]` python package installed.
    For more information, see https://mlflow.org/docs/latest/llms/deployments/server.html.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import MlflowEmbeddings
            embeddings = MlflowEmbeddings(
                target_uri="http://localhost:5000",
                endpoint="embeddings",
            )
    """
    endpoint: str
    """The endpoint to use."""
    target_uri: str
    """The target URI to use."""
    _client: Any = PrivateAttr()
    def __init__(self, **kwargs: Any):
        super().__init__(**kwargs)
        self._validate_uri()
        try:
            from mlflow.deployments import get_deploy_client
            self._client = get_deploy_client(self.target_uri)
        except ImportError as e:
            raise ImportError(
                "Failed to create the client. "
                f"Please run `pip install mlflow{self._mlflow_extras}` to install "
                "required dependencies."
            ) from e
    @property
"required dependencies."
            ) from e
    @property
    def _mlflow_extras(self) -> str:
        return "[genai]"
    def _validate_uri(self) -> None:
        if self.target_uri == "databricks":
            return
        allowed = ["http", "https", "databricks"]
        if urlparse(self.target_uri).scheme not in allowed:
            raise ValueError(
                f"Invalid target URI: {self.target_uri}. "
                f"The scheme must be one of {allowed}."
            )
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        embeddings: List[List[float]] = []
        for txt in _chunk(texts, 20):
            resp = self._client.predict(endpoint=self.endpoint, inputs={"input": txt})
            embeddings.extend(r["embedding"] for r in resp["data"])
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
        return self.embed_documents([text])[0]
Source code for langchain_community.embeddings.huggingface_hub
import json
import os
from typing import Any, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
DEFAULT_MODEL = "sentence-transformers/all-mpnet-base-v2"
VALID_TASKS = ("feature-extraction",)
[docs]class HuggingFaceHubEmbeddings(BaseModel, Embeddings):
    """HuggingFaceHub embedding models.
    To use, you should have the ``huggingface_hub`` python package installed, and the
    environment variable ``HUGGINGFACEHUB_API_TOKEN`` set with your API token, or pass
    it as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import HuggingFaceHubEmbeddings
            model = "sentence-transformers/all-mpnet-base-v2"
            hf = HuggingFaceHubEmbeddings(
                model=model,
                task="feature-extraction",
                huggingfacehub_api_token="my-api-key",
            )
    """
    client: Any  #: :meta private:
    model: Optional[str] = None
    """Model name to use."""
    repo_id: Optional[str] = None
    """Huggingfacehub repository id, for backward compatibility."""
    task: Optional[str] = "feature-extraction"
    """Task to call the model with."""
    model_kwargs: Optional[dict] = None
    """Keyword arguments to pass to the model."""
    huggingfacehub_api_token: Optional[str] = None
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that api key and python package exists in environment."""
        huggingfacehub_api_token = values["huggingfacehub_api_token"] or os.getenv(
            "HUGGINGFACEHUB_API_TOKEN"
        )
        try:
            from huggingface_hub import InferenceClient
            if values["model"]:
                values["repo_id"] = values["model"]
            elif values["repo_id"]:
                values["model"] = values["repo_id"]
            else:
                values["model"] = DEFAULT_MODEL
                values["repo_id"] = DEFAULT_MODEL
            client = InferenceClient(
                model=values["model"],
                token=huggingfacehub_api_token,
            )
            if values["task"] not in VALID_TASKS:
                raise ValueError(
                    f"Got invalid task {values['task']}, "
                    f"currently only {VALID_TASKS} are supported"
                )
            values["client"] = client
        except ImportError:
            raise ImportError(
                "Could not import huggingface_hub python package. "
                "Please install it with `pip install huggingface_hub`."
            )
        return values
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Call out to HuggingFaceHub's embedding endpoint for embedding search docs.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        # replace newlines, which can negatively affect performance.
        texts = [text.replace("\n", " ") for text in texts]
texts = [text.replace("\n", " ") for text in texts]
        _model_kwargs = self.model_kwargs or {}
        responses = self.client.post(
            json={"inputs": texts, "parameters": _model_kwargs}, task=self.task
        )
        return json.loads(responses.decode())
[docs]    def embed_query(self, text: str) -> List[float]:
        """Call out to HuggingFaceHub's embedding endpoint for embedding query text.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        response = self.embed_documents([text])[0]
        return response
Source code for langchain_community.embeddings.clarifai
import logging
from typing import Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
from langchain_core.utils import get_from_dict_or_env
logger = logging.getLogger(__name__)
[docs]class ClarifaiEmbeddings(BaseModel, Embeddings):
    """Clarifai embedding models.
    To use, you should have the ``clarifai`` python package installed, and the
    environment variable ``CLARIFAI_PAT`` set with your personal access token or pass it
    as a named parameter to the constructor.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import ClarifaiEmbeddings
            clarifai = ClarifaiEmbeddings(user_id=USER_ID,
                                          app_id=APP_ID,
                                          model_id=MODEL_ID)
                             (or)
            clarifai_llm = Clarifai(model_url=EXAMPLE_URL)
    """
    model_url: Optional[str] = None
    """Model url to use."""
    model_id: Optional[str] = None
    """Model id to use."""
    model_version_id: Optional[str] = None
    """Model version id to use."""
    app_id: Optional[str] = None
    """Clarifai application id to use."""
    user_id: Optional[str] = None
    """Clarifai user id to use."""
    pat: Optional[str] = None
    """Clarifai personal access token to use."""
    api_base: str = "https://api.clarifai.com"
    class Config:
        """Configuration for this pydantic object."""
        extra = Extra.forbid
    @root_validator()
extra = Extra.forbid
    @root_validator()
    def validate_environment(cls, values: Dict) -> Dict:
        """Validate that we have all required info to access Clarifai
        platform and python package exists in environment."""
        values["pat"] = get_from_dict_or_env(values, "pat", "CLARIFAI_PAT")
        user_id = values.get("user_id")
        app_id = values.get("app_id")
        model_id = values.get("model_id")
        model_url = values.get("model_url")
        if model_url is not None and model_id is not None:
            raise ValueError("Please provide either model_url or model_id, not both.")
        if model_url is None and model_id is None:
            raise ValueError("Please provide one of model_url or model_id.")
        if model_url is None and model_id is not None:
            if user_id is None or app_id is None:
                raise ValueError("Please provide a user_id and app_id.")
        return values
[docs]    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        """Call out to Clarifai's embedding models.
        Args:
            texts: The list of texts to embed.
        Returns:
            List of embeddings, one for each text.
        """
        try:
            from clarifai.client.input import Inputs
            from clarifai.client.model import Model
        except ImportError:
            raise ImportError(
                "Could not import clarifai python package. "
                "Please install it with `pip install clarifai`."
            )
        if self.pat is not None:
            pat = self.pat
        if self.model_url is not None:
            _model_init = Model(url=self.model_url, pat=pat)
_model_init = Model(url=self.model_url, pat=pat)
        else:
            _model_init = Model(
                model_id=self.model_id,
                user_id=self.user_id,
                app_id=self.app_id,
                pat=pat,
            )
        input_obj = Inputs(pat=pat)
        batch_size = 32
        embeddings = []
        try:
            for i in range(0, len(texts), batch_size):
                batch = texts[i : i + batch_size]
                input_batch = [
                    input_obj.get_text_input(input_id=str(id), raw_text=inp)
                    for id, inp in enumerate(batch)
                ]
                predict_response = _model_init.predict(input_batch)
                embeddings.extend(
                    [
                        list(output.data.embeddings[0].vector)
                        for output in predict_response.outputs
                    ]
                )
        except Exception as e:
            logger.error(f"Predict failed, exception: {e}")
        return embeddings
[docs]    def embed_query(self, text: str) -> List[float]:
        """Call out to Clarifai's embedding models.
        Args:
            text: The text to embed.
        Returns:
            Embeddings for the text.
        """
        try:
            from clarifai.client.model import Model
        except ImportError:
            raise ImportError(
                "Could not import clarifai python package. "
                "Please install it with `pip install clarifai`."
            )
        if self.pat is not None:
            pat = self.pat
        if self.model_url is not None:
            _model_init = Model(url=self.model_url, pat=pat)
        else:
            _model_init = Model(
                model_id=self.model_id,
else:
            _model_init = Model(
                model_id=self.model_id,
                user_id=self.user_id,
                app_id=self.app_id,
                pat=pat,
            )
        try:
            predict_response = _model_init.predict_by_bytes(
                bytes(text, "utf-8"), input_type="text"
            )
            embeddings = [
                list(op.data.embeddings[0].vector) for op in predict_response.outputs
            ]
        except Exception as e:
            logger.error(f"Predict failed, exception: {e}")
        return embeddings[0]
Source code for langchain_community.embeddings.ollama
import logging
from typing import Any, Dict, List, Mapping, Optional
import requests
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseModel, Extra
logger = logging.getLogger(__name__)
[docs]class OllamaEmbeddings(BaseModel, Embeddings):
    """Ollama locally runs large language models.
    To use, follow the instructions at https://ollama.ai/.
    Example:
        .. code-block:: python
            from langchain_community.embeddings import OllamaEmbeddings
            ollama_emb = OllamaEmbeddings(
                model="llama:7b",
            )
            r1 = ollama_emb.embed_documents(
                [
                    "Alpha is the first letter of Greek alphabet",
                    "Beta is the second letter of Greek alphabet",
                ]
            )
            r2 = ollama_emb.embed_query(
                "What is the second letter of Greek alphabet"
            )
    """
    base_url: str = "http://localhost:11434"
    """Base url the model is hosted under."""
    model: str = "llama2"
    """Model name to use."""
    embed_instruction: str = "passage: "
    """Instruction used to embed documents."""
    query_instruction: str = "query: "
    """Instruction used to embed the query."""
    mirostat: Optional[int] = None
    """Enable Mirostat sampling for controlling perplexity.
    (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)"""
    mirostat_eta: Optional[float] = None
    """Influences how quickly the algorithm responds to feedback
"""Influences how quickly the algorithm responds to feedback
    from the generated text. A lower learning rate will result in
    slower adjustments, while a higher learning rate will make
    the algorithm more responsive. (Default: 0.1)"""
    mirostat_tau: Optional[float] = None
    """Controls the balance between coherence and diversity
    of the output. A lower value will result in more focused and
    coherent text. (Default: 5.0)"""
    num_ctx: Optional[int] = None
    """Sets the size of the context window used to generate the
    next token. (Default: 2048)	"""
    num_gpu: Optional[int] = None
    """The number of GPUs to use. On macOS it defaults to 1 to
    enable metal support, 0 to disable."""
    num_thread: Optional[int] = None
    """Sets the number of threads to use during computation.
    By default, Ollama will detect this for optimal performance.
    It is recommended to set this value to the number of physical
    CPU cores your system has (as opposed to the logical number of cores)."""
    repeat_last_n: Optional[int] = None
    """Sets how far back for the model to look back to prevent
    repetition. (Default: 64, 0 = disabled, -1 = num_ctx)"""
    repeat_penalty: Optional[float] = None
    """Sets how strongly to penalize repetitions. A higher value (e.g., 1.5)
    will penalize repetitions more strongly, while a lower value (e.g., 0.9)
    will be more lenient. (Default: 1.1)"""
    temperature: Optional[float] = None
    """The temperature of the model. Increasing the temperature will