pinecone = Pinecone.from_texts(
                    texts,
                    embeddings,
                    index_name="langchain-demo"
                )
        """
        try:
            import pinecone
        except ImportError:
            raise ValueError(
                "Could not import pinecone python package. "
                "Please install it with `pip install pinecone-client`."
            )
        indexes = pinecone.list_indexes()  # checks if provided index exists
        if index_name in indexes:
            index = pinecone.Index(index_name)
        elif len(indexes) == 0:
            raise ValueError(
                "No active indexes found in your Pinecone project, "
                "are you sure you're using the right API key and environment?"
            )
        else:
            raise ValueError(
                f"Index '{index_name}' not found in your Pinecone project. "
                f"Did you mean one of the following indexes: {', '.join(indexes)}"
            )
        for i in range(0, len(texts), batch_size):
            # set end position of batch
            i_end = min(i + batch_size, len(texts))
            # get batch of texts and ids
            lines_batch = texts[i:i_end]
            # create ids if not provided
            if ids:
                ids_batch = ids[i:i_end]
            else:
                ids_batch = [str(uuid.uuid4()) for n in range(i, i_end)]
            # create embeddings
            embeds = embedding.embed_documents(lines_batch)
            # prep metadata and upsert batch
            if metadatas:
                metadata = metadatas[i:i_end]
            else:
                metadata = [{} for _ in range(i, i_end)]
            for j, line in enumerate(lines_batch):


for j, line in enumerate(lines_batch):
                metadata[j][text_key] = line
            to_upsert = zip(ids_batch, embeds, metadata)
            # upsert to Pinecone
            index.upsert(vectors=list(to_upsert), namespace=namespace)
        return cls(index, embedding.embed_query, text_key, namespace)
[docs]    @classmethod
    def from_existing_index(
        cls,
        index_name: str,
        embedding: Embeddings,
        text_key: str = "text",
        namespace: Optional[str] = None,
    ) -> Pinecone:
        """Load pinecone vectorstore from index name."""
        try:
            import pinecone
        except ImportError:
            raise ValueError(
                "Could not import pinecone python package. "
                "Please install it with `pip install pinecone-client`."
            )
        return cls(
            pinecone.Index(index_name), embedding.embed_query, text_key, namespace
        )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.vectara
"""Wrapper around Vectara vector database."""
from __future__ import annotations
import json
import logging
import os
from hashlib import md5
from typing import Any, Iterable, List, Optional, Tuple, Type
import requests
from pydantic import Field
from langchain.embeddings.base import Embeddings
from langchain.schema import Document
from langchain.vectorstores.base import VectorStore, VectorStoreRetriever
[docs]class Vectara(VectorStore):
    """Implementation of Vector Store using Vectara (https://vectara.com).
    Example:
        .. code-block:: python
            from langchain.vectorstores import Vectara
            vectorstore = Vectara(
                vectara_customer_id=vectara_customer_id,
                vectara_corpus_id=vectara_corpus_id,
                vectara_api_key=vectara_api_key
            )
    """
    def __init__(
        self,
        vectara_customer_id: Optional[str] = None,
        vectara_corpus_id: Optional[str] = None,
        vectara_api_key: Optional[str] = None,
    ):
        """Initialize with Vectara API."""
        self._vectara_customer_id = vectara_customer_id or os.environ.get(
            "VECTARA_CUSTOMER_ID"
        )
        self._vectara_corpus_id = vectara_corpus_id or os.environ.get(
            "VECTARA_CORPUS_ID"
        )
        self._vectara_api_key = vectara_api_key or os.environ.get("VECTARA_API_KEY")
        if (
            self._vectara_customer_id is None
            or self._vectara_corpus_id is None
            or self._vectara_api_key is None
        ):
            logging.warning(


or self._vectara_api_key is None
        ):
            logging.warning(
                "Cant find Vectara credentials, customer_id or corpus_id in "
                "environment."
            )
        else:
            logging.debug(f"Using corpus id {self._vectara_corpus_id}")
        self._session = requests.Session()  # to reuse connections
    def _get_post_headers(self) -> dict:
        """Returns headers that should be attached to each post request."""
        return {
            "x-api-key": self._vectara_api_key,
            "customer-id": self._vectara_customer_id,
            "Content-Type": "application/json",
        }
    def _delete_doc(self, doc_id: str) -> bool:
        """
        Delete a document from the Vectara corpus.
        Args:
            url (str): URL of the page to delete.
            doc_id (str): ID of the document to delete.
        Returns:
            bool: True if deletion was successful, False otherwise.
        """
        body = {
            "customer_id": self._vectara_customer_id,
            "corpus_id": self._vectara_corpus_id,
            "document_id": doc_id,
        }
        response = self._session.post(
            "https://api.vectara.io/v1/delete-doc",
            data=json.dumps(body),
            verify=True,
            headers=self._get_post_headers(),
        )
        if response.status_code != 200:
            logging.error(
                f"Delete request failed for doc_id = {doc_id} with status code "
                f"{response.status_code}, reason {response.reason}, text "
                f"{response.text}"
            )
            return False
        return True


f"{response.text}"
            )
            return False
        return True
    def _index_doc(self, doc_id: str, text: str, metadata: dict) -> bool:
        request: dict[str, Any] = {}
        request["customer_id"] = self._vectara_customer_id
        request["corpus_id"] = self._vectara_corpus_id
        request["document"] = {
            "document_id": doc_id,
            "metadataJson": json.dumps(metadata),
            "section": [{"text": text, "metadataJson": json.dumps(metadata)}],
        }
        response = self._session.post(
            headers=self._get_post_headers(),
            url="https://api.vectara.io/v1/index",
            data=json.dumps(request),
            timeout=30,
            verify=True,
        )
        status_code = response.status_code
        result = response.json()
        status_str = result["status"]["code"] if "status" in result else None
        if status_code == 409 or (status_str and status_str == "ALREADY_EXISTS"):
            return False
        else:
            return True
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        ids = [md5(text.encode("utf-8")).hexdigest() for text in texts]


for i, doc in enumerate(texts):
            doc_id = ids[i]
            metadata = metadatas[i] if metadatas else {}
            succeeded = self._index_doc(doc_id, doc, metadata)
            if not succeeded:
                self._delete_doc(doc_id)
                self._index_doc(doc_id, doc, metadata)
        return ids
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 5,
        alpha: float = 0.025,
        filter: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return Vectara documents most similar to query, along with scores.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 5.
            alpha: parameter for hybrid search (called "lambda" in Vectara
                documentation).
            filter: Dictionary of argument(s) to filter on metadata. For example a
                filter can be "doc.rating > 3.0 and part.lang = 'deu'"} see
                https://docs.vectara.com/docs/search-apis/sql/filter-overview
                for more details.
        Returns:
            List of Documents most similar to the query and score for each.
        """
        response = self._session.post(
            headers=self._get_post_headers(),
            url="https://api.vectara.io/v1/query",
            data=json.dumps(
                {
                    "query": [
                        {
                            "query": query,
                            "start": 0,
                            "num_results": k,
                            "context_config": {
                                "sentences_before": 3,


"context_config": {
                                "sentences_before": 3,
                                "sentences_after": 3,
                            },
                            "corpus_key": [
                                {
                                    "customer_id": self._vectara_customer_id,
                                    "corpus_id": self._vectara_corpus_id,
                                    "metadataFilter": filter,
                                    "lexical_interpolation_config": {"lambda": alpha},
                                }
                            ],
                        }
                    ]
                }
            ),
            timeout=10,
        )
        if response.status_code != 200:
            logging.error(
                "Query failed %s",
                f"(code {response.status_code}, reason {response.reason}, details "
                f"{response.text})",
            )
            return []
        result = response.json()
        responses = result["responseSet"][0]["response"]
        vectara_default_metadata = ["lang", "len", "offset"]
        docs = [
            (
                Document(
                    page_content=x["text"],
                    metadata={
                        m["name"]: m["value"]
                        for m in x["metadata"]
                        if m["name"] not in vectara_default_metadata
                    },
                ),
                x["score"],
            )
            for x in responses
        ]
        return docs
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 5,
        alpha: float = 0.025,
        filter: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return Vectara documents most similar to query, along with scores.
        Args:
            query: Text to look up documents similar to.


Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 5.
            filter: Dictionary of argument(s) to filter on metadata. For example a
                filter can be "doc.rating > 3.0 and part.lang = 'deu'"} see
                https://docs.vectara.com/docs/search-apis/sql/filter-overview for more
                details.
        Returns:
            List of Documents most similar to the query
        """
        docs_and_scores = self.similarity_search_with_score(
            query, k=k, alpha=alpha, filter=filter, **kwargs
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    @classmethod
    def from_texts(
        cls: Type[Vectara],
        texts: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> Vectara:
        """Construct Vectara wrapper from raw documents.
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain import Vectara
                vectara = Vectara.from_texts(
                    texts,
                    vectara_customer_id=customer_id,
                    vectara_corpus_id=corpus_id,
                    vectara_api_key=api_key,
                )
        """
        # Note: Vectara generates its own embeddings, so we ignore the provided
        # embeddings (required by interface)
        vectara = cls(**kwargs)
        vectara.add_texts(texts, metadatas)
        return vectara
[docs]    def as_retriever(self, **kwargs: Any) -> VectaraRetriever:


return VectaraRetriever(vectorstore=self, **kwargs)
class VectaraRetriever(VectorStoreRetriever):
    vectorstore: Vectara
    search_kwargs: dict = Field(default_factory=lambda: {"alpha": 0.025, "k": 5})
    """Search params.
        k: Number of Documents to return. Defaults to 5.
        alpha: parameter for hybrid search (called "lambda" in Vectara
            documentation).
        filter: Dictionary of argument(s) to filter on metadata. For example a
            filter can be "doc.rating > 3.0 and part.lang = 'deu'"} see
            https://docs.vectara.com/docs/search-apis/sql/filter-overview
            for more details.
    """
    def add_texts(
        self, texts: List[str], metadatas: Optional[List[dict]] = None
    ) -> None:
        """Add text to the Vectara vectorstore.
        Args:
            texts (List[str]): The text
            metadatas (List[dict]): Metadata dicts, must line up with existing store
        """
        self.vectorstore.add_texts(texts, metadatas)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.redis
"""Wrapper around Redis vector database."""
from __future__ import annotations
import json
import logging
import uuid
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Literal,
    Mapping,
    Optional,
    Tuple,
    Type,
)
import numpy as np
from pydantic import BaseModel, root_validator
from langchain.docstore.document import Document
from langchain.embeddings.base import Embeddings
from langchain.utils import get_from_dict_or_env
from langchain.vectorstores.base import VectorStore, VectorStoreRetriever
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
    from redis.client import Redis as RedisType
    from redis.commands.search.query import Query
# required modules
REDIS_REQUIRED_MODULES = [
    {"name": "search", "ver": 20400},
    {"name": "searchlight", "ver": 20400},
]
# distance mmetrics
REDIS_DISTANCE_METRICS = Literal["COSINE", "IP", "L2"]
def _check_redis_module_exist(client: RedisType, required_modules: List[dict]) -> None:
    """Check if the correct Redis modules are installed."""
    installed_modules = client.module_list()
    installed_modules = {
        module[b"name"].decode("utf-8"): module for module in installed_modules
    }
    for module in required_modules:
        if module["name"] in installed_modules and int(
            installed_modules[module["name"]][b"ver"]
        ) >= int(module["ver"]):
            return
    # otherwise raise error
    error_message = (
        "Redis cannot be used as a vector database without RediSearch >=2.4"


"Redis cannot be used as a vector database without RediSearch >=2.4"
        "Please head to https://redis.io/docs/stack/search/quick_start/"
        "to know more about installing the RediSearch module within Redis Stack."
    )
    logging.error(error_message)
    raise ValueError(error_message)
def _check_index_exists(client: RedisType, index_name: str) -> bool:
    """Check if Redis index exists."""
    try:
        client.ft(index_name).info()
    except:  # noqa: E722
        logger.info("Index does not exist")
        return False
    logger.info("Index already exists")
    return True
def _redis_key(prefix: str) -> str:
    """Redis key schema for a given prefix."""
    return f"{prefix}:{uuid.uuid4().hex}"
def _redis_prefix(index_name: str) -> str:
    """Redis key prefix for a given index."""
    return f"doc:{index_name}"
def _default_relevance_score(val: float) -> float:
    return 1 - val
[docs]class Redis(VectorStore):
    """Wrapper around Redis vector database.
    To use, you should have the ``redis`` python package installed.
    Example:
        .. code-block:: python
            from langchain.vectorstores import Redis
            from langchain.embeddings import OpenAIEmbeddings
            embeddings = OpenAIEmbeddings()
            vectorstore = Redis(
                redis_url="redis://username:password@localhost:6379"
                index_name="my-index",
                embedding_function=embeddings.embed_query,
            )
    """
    def __init__(
        self,
        redis_url: str,
        index_name: str,
        embedding_function: Callable,


index_name: str,
        embedding_function: Callable,
        content_key: str = "content",
        metadata_key: str = "metadata",
        vector_key: str = "content_vector",
        relevance_score_fn: Optional[
            Callable[[float], float]
        ] = _default_relevance_score,
        **kwargs: Any,
    ):
        """Initialize with necessary components."""
        try:
            import redis
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis>=4.1.0`."
            )
        self.embedding_function = embedding_function
        self.index_name = index_name
        try:
            # connect to redis from url
            redis_client = redis.from_url(redis_url, **kwargs)
            # check if redis has redisearch module installed
            _check_redis_module_exist(redis_client, REDIS_REQUIRED_MODULES)
        except ValueError as e:
            raise ValueError(f"Redis failed to connect: {e}")
        self.client = redis_client
        self.content_key = content_key
        self.metadata_key = metadata_key
        self.vector_key = vector_key
        self.relevance_score_fn = relevance_score_fn
    def _create_index(
        self, dim: int = 1536, distance_metric: REDIS_DISTANCE_METRICS = "COSINE"
    ) -> None:
        try:
            from redis.commands.search.field import TextField, VectorField
            from redis.commands.search.indexDefinition import IndexDefinition, IndexType
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        # Check if index exists
        if not _check_index_exists(self.client, self.index_name):


if not _check_index_exists(self.client, self.index_name):
            # Define schema
            schema = (
                TextField(name=self.content_key),
                TextField(name=self.metadata_key),
                VectorField(
                    self.vector_key,
                    "FLAT",
                    {
                        "TYPE": "FLOAT32",
                        "DIM": dim,
                        "DISTANCE_METRIC": distance_metric,
                    },
                ),
            )
            prefix = _redis_prefix(self.index_name)
            # Create Redis Index
            self.client.ft(self.index_name).create_index(
                fields=schema,
                definition=IndexDefinition(prefix=[prefix], index_type=IndexType.HASH),
            )
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        embeddings: Optional[List[List[float]]] = None,
        keys: Optional[List[str]] = None,
        batch_size: int = 1000,
        **kwargs: Any,
    ) -> List[str]:
        """Add more texts to the vectorstore.
        Args:
            texts (Iterable[str]): Iterable of strings/text to add to the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadatas.
                Defaults to None.
            embeddings (Optional[List[List[float]]], optional): Optional pre-generated
                embeddings. Defaults to None.
            keys (Optional[List[str]], optional): Optional key values to use as ids.
                Defaults to None.
            batch_size (int, optional): Batch size to use for writes. Defaults to 1000.
        Returns:
            List[str]: List of ids added to the vectorstore
        """
        ids = []
        prefix = _redis_prefix(self.index_name)


"""
        ids = []
        prefix = _redis_prefix(self.index_name)
        # Write data to redis
        pipeline = self.client.pipeline(transaction=False)
        for i, text in enumerate(texts):
            # Use provided values by default or fallback
            key = keys[i] if keys else _redis_key(prefix)
            metadata = metadatas[i] if metadatas else {}
            embedding = embeddings[i] if embeddings else self.embedding_function(text)
            pipeline.hset(
                key,
                mapping={
                    self.content_key: text,
                    self.vector_key: np.array(embedding, dtype=np.float32).tobytes(),
                    self.metadata_key: json.dumps(metadata),
                },
            )
            ids.append(key)
            # Write batch
            if i % batch_size == 0:
                pipeline.execute()
        # Cleanup final batch
        pipeline.execute()
        return ids
[docs]    def similarity_search(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """
        Returns the most similar indexed documents to the query text.
        Args:
            query (str): The query text for which to find similar documents.
            k (int): The number of documents to return. Default is 4.
        Returns:
            List[Document]: A list of documents that are most similar to the query text.
        """
        docs_and_scores = self.similarity_search_with_score(query, k=k)
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_limit_score(
        self, query: str, k: int = 4, score_threshold: float = 0.2, **kwargs: Any
    ) -> List[Document]:
        """


) -> List[Document]:
        """
        Returns the most similar indexed documents to the query text within the
        score_threshold range.
        Args:
            query (str): The query text for which to find similar documents.
            k (int): The number of documents to return. Default is 4.
            score_threshold (float): The minimum matching score required for a document
            to be considered a match. Defaults to 0.2.
            Because the similarity calculation algorithm is based on cosine similarity,
            the smaller the angle, the higher the similarity.
        Returns:
            List[Document]: A list of documents that are most similar to the query text,
            including the match score for each document.
        Note:
            If there are no documents that satisfy the score_threshold value,
            an empty list is returned.
        """
        docs_and_scores = self.similarity_search_with_score(query, k=k)
        return [doc for doc, score in docs_and_scores if score < score_threshold]
    def _prepare_query(self, k: int) -> Query:
        try:
            from redis.commands.search.query import Query
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        # Prepare the Query
        hybrid_fields = "*"
        base_query = (
            f"{hybrid_fields}=>[KNN {k} @{self.vector_key} $vector AS vector_score]"
        )
        return_fields = [self.metadata_key, self.content_key, "vector_score"]
        return (
            Query(base_query)
            .return_fields(*return_fields)
            .sort_by("vector_score")
            .paging(0, k)
            .dialect(2)
        )


.paging(0, k)
            .dialect(2)
        )
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of Documents most similar to the query and score for each
        """
        # Creates embedding vector from user query
        embedding = self.embedding_function(query)
        # Creates Redis query
        redis_query = self._prepare_query(k)
        params_dict: Mapping[str, str] = {
            "vector": np.array(embedding)  # type: ignore
            .astype(dtype=np.float32)
            .tobytes()
        }
        # Perform vector search
        results = self.client.ft(self.index_name).search(redis_query, params_dict)
        # Prepare document results
        docs = [
            (
                Document(
                    page_content=result.content, metadata=json.loads(result.metadata)
                ),
                float(result.vector_score),
            )
            for result in results.docs
        ]
        return docs
    def _similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return docs and relevance scores, normalized on a scale from 0 to 1.
        0 is dissimilar, 1 is most similar.
        """
        if self.relevance_score_fn is None:
            raise ValueError(
                "relevance_score_fn must be provided to"


raise ValueError(
                "relevance_score_fn must be provided to"
                " Redis constructor to normalize scores"
            )
        docs_and_scores = self.similarity_search_with_score(query, k=k)
        return [(doc, self.relevance_score_fn(score)) for doc, score in docs_and_scores]
[docs]    @classmethod
    def from_texts_return_keys(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        index_name: Optional[str] = None,
        content_key: str = "content",
        metadata_key: str = "metadata",
        vector_key: str = "content_vector",
        distance_metric: REDIS_DISTANCE_METRICS = "COSINE",
        **kwargs: Any,
    ) -> Tuple[Redis, List[str]]:
        """Create a Redis vectorstore from raw documents.
        This is a user-friendly interface that:
            1. Embeds documents.
            2. Creates a new index for the embeddings in Redis.
            3. Adds the documents to the newly created Redis index.
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Redis
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                redisearch = RediSearch.from_texts(
                    texts,
                    embeddings,
                    redis_url="redis://username:password@localhost:6379"
                )
        """
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        if "redis_url" in kwargs:
            kwargs.pop("redis_url")
        # Name of the search index if not given


kwargs.pop("redis_url")
        # Name of the search index if not given
        if not index_name:
            index_name = uuid.uuid4().hex
        # Create instance
        instance = cls(
            redis_url,
            index_name,
            embedding.embed_query,
            content_key=content_key,
            metadata_key=metadata_key,
            vector_key=vector_key,
            **kwargs,
        )
        # Create embeddings over documents
        embeddings = embedding.embed_documents(texts)
        # Create the search index
        instance._create_index(dim=len(embeddings[0]), distance_metric=distance_metric)
        # Add data to Redis
        keys = instance.add_texts(texts, metadatas, embeddings)
        return instance, keys
[docs]    @classmethod
    def from_texts(
        cls: Type[Redis],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        index_name: Optional[str] = None,
        content_key: str = "content",
        metadata_key: str = "metadata",
        vector_key: str = "content_vector",
        **kwargs: Any,
    ) -> Redis:
        """Create a Redis vectorstore from raw documents.
        This is a user-friendly interface that:
            1. Embeds documents.
            2. Creates a new index for the embeddings in Redis.
            3. Adds the documents to the newly created Redis index.
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain.vectorstores import Redis
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                redisearch = RediSearch.from_texts(


embeddings = OpenAIEmbeddings()
                redisearch = RediSearch.from_texts(
                    texts,
                    embeddings,
                    redis_url="redis://username:password@localhost:6379"
                )
        """
        instance, _ = cls.from_texts_return_keys(
            texts,
            embedding,
            metadatas=metadatas,
            index_name=index_name,
            content_key=content_key,
            metadata_key=metadata_key,
            vector_key=vector_key,
            **kwargs,
        )
        return instance
[docs]    @staticmethod
    def drop_index(
        index_name: str,
        delete_documents: bool,
        **kwargs: Any,
    ) -> bool:
        """
        Drop a Redis search index.
        Args:
            index_name (str): Name of the index to drop.
            delete_documents (bool): Whether to drop the associated documents.
        Returns:
            bool: Whether or not the drop was successful.
        """
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        try:
            import redis
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        try:
            # We need to first remove redis_url from kwargs,
            # otherwise passing it to Redis will result in an error.
            if "redis_url" in kwargs:
                kwargs.pop("redis_url")
            client = redis.from_url(url=redis_url, **kwargs)
        except ValueError as e:
            raise ValueError(f"Your redis connected error: {e}")
        # Check if index exists
        try:
            client.ft(index_name).dropindex(delete_documents)


try:
            client.ft(index_name).dropindex(delete_documents)
            logger.info("Drop index")
            return True
        except:  # noqa: E722
            # Index not exist
            return False
[docs]    @classmethod
    def from_existing_index(
        cls,
        embedding: Embeddings,
        index_name: str,
        content_key: str = "content",
        metadata_key: str = "metadata",
        vector_key: str = "content_vector",
        **kwargs: Any,
    ) -> Redis:
        """Connect to an existing Redis index."""
        redis_url = get_from_dict_or_env(kwargs, "redis_url", "REDIS_URL")
        try:
            import redis
        except ImportError:
            raise ValueError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        try:
            # We need to first remove redis_url from kwargs,
            # otherwise passing it to Redis will result in an error.
            if "redis_url" in kwargs:
                kwargs.pop("redis_url")
            client = redis.from_url(url=redis_url, **kwargs)
            # check if redis has redisearch module installed
            _check_redis_module_exist(client, REDIS_REQUIRED_MODULES)
            # ensure that the index already exists
            assert _check_index_exists(
                client, index_name
            ), f"Index {index_name} does not exist"
        except Exception as e:
            raise ValueError(f"Redis failed to connect: {e}")
        return cls(
            redis_url,
            index_name,
            embedding.embed_query,
            content_key=content_key,
            metadata_key=metadata_key,
            vector_key=vector_key,
            **kwargs,
        )


vector_key=vector_key,
            **kwargs,
        )
[docs]    def as_retriever(self, **kwargs: Any) -> RedisVectorStoreRetriever:
        return RedisVectorStoreRetriever(vectorstore=self, **kwargs)
class RedisVectorStoreRetriever(VectorStoreRetriever, BaseModel):
    vectorstore: Redis
    search_type: str = "similarity"
    k: int = 4
    score_threshold: float = 0.4
    class Config:
        """Configuration for this pydantic object."""
        arbitrary_types_allowed = True
    @root_validator()
    def validate_search_type(cls, values: Dict) -> Dict:
        """Validate search type."""
        if "search_type" in values:
            search_type = values["search_type"]
            if search_type not in ("similarity", "similarity_limit"):
                raise ValueError(f"search_type of {search_type} not allowed.")
        return values
    def get_relevant_documents(self, query: str) -> List[Document]:
        if self.search_type == "similarity":
            docs = self.vectorstore.similarity_search(query, k=self.k)
        elif self.search_type == "similarity_limit":
            docs = self.vectorstore.similarity_search_limit_score(
                query, k=self.k, score_threshold=self.score_threshold
            )
        else:
            raise ValueError(f"search_type of {self.search_type} not allowed.")
        return docs
    async def aget_relevant_documents(self, query: str) -> List[Document]:
        raise NotImplementedError("RedisVectorStoreRetriever does not support async")
    def add_documents(self, documents: List[Document], **kwargs: Any) -> List[str]:
        """Add documents to vectorstore."""


"""Add documents to vectorstore."""
        return self.vectorstore.add_documents(documents, **kwargs)
    async def aadd_documents(
        self, documents: List[Document], **kwargs: Any
    ) -> List[str]:
        """Add documents to vectorstore."""
        return await self.vectorstore.aadd_documents(documents, **kwargs)
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.atlas
"""Wrapper around Atlas by Nomic."""
from __future__ import annotations
import logging
import uuid
from typing import Any, Iterable, List, Optional, Type
import numpy as np
from langchain.docstore.document import Document
from langchain.embeddings.base import Embeddings
from langchain.vectorstores.base import VectorStore
logger = logging.getLogger(__name__)
[docs]class AtlasDB(VectorStore):
    """Wrapper around Atlas: Nomic's neural database and rhizomatic instrument.
    To use, you should have the ``nomic`` python package installed.
    Example:
        .. code-block:: python
                from langchain.vectorstores import AtlasDB
                from langchain.embeddings.openai import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                vectorstore = AtlasDB("my_project", embeddings.embed_query)
    """
    _ATLAS_DEFAULT_ID_FIELD = "atlas_id"
    def __init__(
        self,
        name: str,
        embedding_function: Optional[Embeddings] = None,
        api_key: Optional[str] = None,
        description: str = "A description for your project",
        is_public: bool = True,
        reset_project_if_exists: bool = False,
    ) -> None:
        """
        Initialize the Atlas Client
        Args:
            name (str): The name of your project. If the project already exists,
                it will be loaded.
            embedding_function (Optional[Callable]): An optional function used for
                embedding your data. If None, data will be embedded with
                Nomic's embed model.
            api_key (str): Your nomic API key
            description (str): A description for your project.
            is_public (bool): Whether your project is publicly accessible.


is_public (bool): Whether your project is publicly accessible.
                True by default.
            reset_project_if_exists (bool): Whether to reset this project if it
                already exists. Default False.
                Generally userful during development and testing.
        """
        try:
            import nomic
            from nomic import AtlasProject
        except ImportError:
            raise ValueError(
                "Could not import nomic python package. "
                "Please install it with `pip install nomic`."
            )
        if api_key is None:
            raise ValueError("No API key provided. Sign up at atlas.nomic.ai!")
        nomic.login(api_key)
        self._embedding_function = embedding_function
        modality = "text"
        if self._embedding_function is not None:
            modality = "embedding"
        # Check if the project exists, create it if not
        self.project = AtlasProject(
            name=name,
            description=description,
            modality=modality,
            is_public=is_public,
            reset_project_if_exists=reset_project_if_exists,
            unique_id_field=AtlasDB._ATLAS_DEFAULT_ID_FIELD,
        )
        self.project._latest_project_state()
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        refresh: bool = True,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Args:
            texts (Iterable[str]): Texts to add to the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadatas.


metadatas (Optional[List[dict]], optional): Optional list of metadatas.
            ids (Optional[List[str]]): An optional list of ids.
            refresh(bool): Whether or not to refresh indices with the updated data.
                Default True.
        Returns:
            List[str]: List of IDs of the added texts.
        """
        if (
            metadatas is not None
            and len(metadatas) > 0
            and "text" in metadatas[0].keys()
        ):
            raise ValueError("Cannot accept key text in metadata!")
        texts = list(texts)
        if ids is None:
            ids = [str(uuid.uuid1()) for _ in texts]
        # Embedding upload case
        if self._embedding_function is not None:
            _embeddings = self._embedding_function.embed_documents(texts)
            embeddings = np.stack(_embeddings)
            if metadatas is None:
                data = [
                    {AtlasDB._ATLAS_DEFAULT_ID_FIELD: ids[i], "text": texts[i]}
                    for i, _ in enumerate(texts)
                ]
            else:
                for i in range(len(metadatas)):
                    metadatas[i][AtlasDB._ATLAS_DEFAULT_ID_FIELD] = ids[i]
                    metadatas[i]["text"] = texts[i]
                data = metadatas
            self.project._validate_map_data_inputs(
                [], id_field=AtlasDB._ATLAS_DEFAULT_ID_FIELD, data=data
            )
            with self.project.wait_for_project_lock():
                self.project.add_embeddings(embeddings=embeddings, data=data)
        # Text upload case
        else:
            if metadatas is None:
                data = [


else:
            if metadatas is None:
                data = [
                    {"text": text, AtlasDB._ATLAS_DEFAULT_ID_FIELD: ids[i]}
                    for i, text in enumerate(texts)
                ]
            else:
                for i, text in enumerate(texts):
                    metadatas[i]["text"] = texts
                    metadatas[i][AtlasDB._ATLAS_DEFAULT_ID_FIELD] = ids[i]
                data = metadatas
            self.project._validate_map_data_inputs(
                [], id_field=AtlasDB._ATLAS_DEFAULT_ID_FIELD, data=data
            )
            with self.project.wait_for_project_lock():
                self.project.add_text(data)
        if refresh:
            if len(self.project.indices) > 0:
                with self.project.wait_for_project_lock():
                    self.project.rebuild_maps()
        return ids
[docs]    def create_index(self, **kwargs: Any) -> Any:
        """Creates an index in your project.
        See
        https://docs.nomic.ai/atlas_api.html#nomic.project.AtlasProject.create_index
        for full detail.
        """
        with self.project.wait_for_project_lock():
            return self.project.create_index(**kwargs)
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Document]:
        """Run similarity search with AtlasDB
        Args:
            query (str): Query text to search for.
            k (int): Number of results to return. Defaults to 4.
        Returns:
            List[Document]: List of documents most similar to the query text.
        """
        if self._embedding_function is None:
            raise NotImplementedError(


"""
        if self._embedding_function is None:
            raise NotImplementedError(
                "AtlasDB requires an embedding_function for text similarity search!"
            )
        _embedding = self._embedding_function.embed_documents([query])[0]
        embedding = np.array(_embedding).reshape(1, -1)
        with self.project.wait_for_project_lock():
            neighbors, _ = self.project.projections[0].vector_search(
                queries=embedding, k=k
            )
            datas = self.project.get_data(ids=neighbors[0])
        docs = [
            Document(page_content=datas[i]["text"], metadata=datas[i])
            for i, neighbor in enumerate(neighbors)
        ]
        return docs
[docs]    @classmethod
    def from_texts(
        cls: Type[AtlasDB],
        texts: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        name: Optional[str] = None,
        api_key: Optional[str] = None,
        description: str = "A description for your project",
        is_public: bool = True,
        reset_project_if_exists: bool = False,
        index_kwargs: Optional[dict] = None,
        **kwargs: Any,
    ) -> AtlasDB:
        """Create an AtlasDB vectorstore from a raw documents.
        Args:
            texts (List[str]): The list of texts to ingest.
            name (str): Name of the project to create.
            api_key (str): Your nomic API key,
            embedding (Optional[Embeddings]): Embedding function. Defaults to None.
            metadatas (Optional[List[dict]]): List of metadatas. Defaults to None.


ids (Optional[List[str]]): Optional list of document IDs. If None,
                ids will be auto created
            description (str): A description for your project.
            is_public (bool): Whether your project is publicly accessible.
                True by default.
            reset_project_if_exists (bool): Whether to reset this project if it
                already exists. Default False.
                Generally userful during development and testing.
            index_kwargs (Optional[dict]): Dict of kwargs for index creation.
                See https://docs.nomic.ai/atlas_api.html
        Returns:
            AtlasDB: Nomic's neural database and finest rhizomatic instrument
        """
        if name is None or api_key is None:
            raise ValueError("`name` and `api_key` cannot be None.")
        # Inject relevant kwargs
        all_index_kwargs = {"name": name + "_index", "indexed_field": "text"}
        if index_kwargs is not None:
            for k, v in index_kwargs.items():
                all_index_kwargs[k] = v
        # Build project
        atlasDB = cls(
            name,
            embedding_function=embedding,
            api_key=api_key,
            description="A description for your project",
            is_public=is_public,
            reset_project_if_exists=reset_project_if_exists,
        )
        with atlasDB.project.wait_for_project_lock():
            atlasDB.add_texts(texts=texts, metadatas=metadatas, ids=ids)
            atlasDB.create_index(**all_index_kwargs)
        return atlasDB
[docs]    @classmethod
    def from_documents(
        cls: Type[AtlasDB],
        documents: List[Document],
        embedding: Optional[Embeddings] = None,
        ids: Optional[List[str]] = None,


ids: Optional[List[str]] = None,
        name: Optional[str] = None,
        api_key: Optional[str] = None,
        persist_directory: Optional[str] = None,
        description: str = "A description for your project",
        is_public: bool = True,
        reset_project_if_exists: bool = False,
        index_kwargs: Optional[dict] = None,
        **kwargs: Any,
    ) -> AtlasDB:
        """Create an AtlasDB vectorstore from a list of documents.
        Args:
            name (str): Name of the collection to create.
            api_key (str): Your nomic API key,
            documents (List[Document]): List of documents to add to the vectorstore.
            embedding (Optional[Embeddings]): Embedding function. Defaults to None.
            ids (Optional[List[str]]): Optional list of document IDs. If None,
                ids will be auto created
            description (str): A description for your project.
            is_public (bool): Whether your project is publicly accessible.
                True by default.
            reset_project_if_exists (bool): Whether to reset this project if
                it already exists. Default False.
                Generally userful during development and testing.
            index_kwargs (Optional[dict]): Dict of kwargs for index creation.
                See https://docs.nomic.ai/atlas_api.html
        Returns:
            AtlasDB: Nomic's neural database and finest rhizomatic instrument
        """
        if name is None or api_key is None:
            raise ValueError("`name` and `api_key` cannot be None.")
        texts = [doc.page_content for doc in documents]
        metadatas = [doc.metadata for doc in documents]
        return cls.from_texts(
            name=name,
            api_key=api_key,


return cls.from_texts(
            name=name,
            api_key=api_key,
            texts=texts,
            embedding=embedding,
            metadatas=metadatas,
            ids=ids,
            description=description,
            is_public=is_public,
            reset_project_if_exists=reset_project_if_exists,
            index_kwargs=index_kwargs,
        )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.annoy
"""Wrapper around Annoy vector database."""
from __future__ import annotations
import os
import pickle
import uuid
from configparser import ConfigParser
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
import numpy as np
from langchain.docstore.base import Docstore
from langchain.docstore.document import Document
from langchain.docstore.in_memory import InMemoryDocstore
from langchain.embeddings.base import Embeddings
from langchain.vectorstores.base import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
INDEX_METRICS = frozenset(["angular", "euclidean", "manhattan", "hamming", "dot"])
DEFAULT_METRIC = "angular"
def dependable_annoy_import() -> Any:
    """Import annoy if available, otherwise raise error."""
    try:
        import annoy
    except ImportError:
        raise ValueError(
            "Could not import annoy python package. "
            "Please install it with `pip install --user annoy` "
        )
    return annoy
[docs]class Annoy(VectorStore):
    """Wrapper around Annoy vector database.
    To use, you should have the ``annoy`` python package installed.
    Example:
        .. code-block:: python
            from langchain import Annoy
            db = Annoy(embedding_function, index, docstore, index_to_docstore_id)
    """
    def __init__(
        self,
        embedding_function: Callable,
        index: Any,
        metric: str,
        docstore: Docstore,
        index_to_docstore_id: Dict[int, str],
    ):
        """Initialize with necessary components."""
        self.embedding_function = embedding_function


):
        """Initialize with necessary components."""
        self.embedding_function = embedding_function
        self.index = index
        self.metric = metric
        self.docstore = docstore
        self.index_to_docstore_id = index_to_docstore_id
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        raise NotImplementedError(
            "Annoy does not allow to add new data once the index is build."
        )
[docs]    def process_index_results(
        self, idxs: List[int], dists: List[float]
    ) -> List[Tuple[Document, float]]:
        """Turns annoy results into a list of documents and scores.
        Args:
            idxs: List of indices of the documents in the index.
            dists: List of distances of the documents in the index.
        Returns:
            List of Documents and scores.
        """
        docs = []
        for idx, dist in zip(idxs, dists):
            _id = self.index_to_docstore_id[idx]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append((doc, dist))
        return docs
[docs]    def similarity_search_with_score_by_vector(
        self, embedding: List[float], k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.


Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        idxs, dists = self.index.get_nns_by_vector(
            embedding, k, search_k=search_k, include_distances=True
        )
        return self.process_index_results(idxs, dists)
[docs]    def similarity_search_with_score_by_index(
        self, docstore_index: int, k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        idxs, dists = self.index.get_nns_by_item(
            docstore_index, k, search_k=search_k, include_distances=True
        )
        return self.process_index_results(idxs, dists)
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.


k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        embedding = self.embedding_function(query)
        docs = self.similarity_search_with_score_by_vector(embedding, k, search_k)
        return docs
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to embedding vector.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_vector(
            embedding, k, search_k
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_by_index(
        self, docstore_index: int, k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to docstore_index.
        Args:
            docstore_index: Index of document in docstore
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the embedding.
        """


Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_index(
            docstore_index, k, search_k
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search(
        self, query: str, k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query.
        """
        docs_and_scores = self.similarity_search_with_score(query, k, search_k)
        return [doc for doc, _ in docs_and_scores]
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            k: Number of Documents to return. Defaults to 4.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding


of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        idxs = self.index.get_nns_by_vector(
            embedding, fetch_k, search_k=-1, include_distances=False
        )
        embeddings = [self.index.get_item_vector(i) for i in idxs]
        mmr_selected = maximal_marginal_relevance(
            np.array([embedding], dtype=np.float32),
            embeddings,
            k=k,
            lambda_mult=lambda_mult,
        )
        # ignore the -1's if not enough docs are returned/indexed
        selected_indices = [idxs[i] for i in mmr_selected if i != -1]
        docs = []
        for i in selected_indices:
            _id = self.index_to_docstore_id[i]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append(doc)
        return docs
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.


k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding = self.embedding_function(query)
        docs = self.max_marginal_relevance_search_by_vector(
            embedding, k, fetch_k, lambda_mult=lambda_mult
        )
        return docs
    @classmethod
    def __from(
        cls,
        texts: List[str],
        embeddings: List[List[float]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        if metric not in INDEX_METRICS:
            raise ValueError(
                (
                    f"Unsupported distance metric: {metric}. "
                    f"Expected one of {list(INDEX_METRICS)}"
                )
            )
        annoy = dependable_annoy_import()
        if not embeddings:
            raise ValueError("embeddings must be provided to build AnnoyIndex")
        f = len(embeddings[0])
        index = annoy.AnnoyIndex(f, metric=metric)
        for i, emb in enumerate(embeddings):
            index.add_item(i, emb)
        index.build(trees, n_jobs=n_jobs)
        documents = []
        for i, text in enumerate(texts):


documents = []
        for i, text in enumerate(texts):
            metadata = metadatas[i] if metadatas else {}
            documents.append(Document(page_content=text, metadata=metadata))
        index_to_id = {i: str(uuid.uuid4()) for i in range(len(documents))}
        docstore = InMemoryDocstore(
            {index_to_id[i]: doc for i, doc in enumerate(documents)}
        )
        return cls(embedding.embed_query, index, metric, docstore, index_to_id)
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        """Construct Annoy wrapper from raw documents.
        Args:
            texts: List of documents to index.
            embedding: Embedding function to use.
            metadatas: List of metadata dictionaries to associate with documents.
            metric: Metric to use for indexing. Defaults to "angular".
            trees: Number of trees to use for indexing. Defaults to 100.
            n_jobs: Number of jobs to use for indexing. Defaults to -1.
        This is a user friendly interface that:
            1. Embeds documents.
            2. Creates an in memory docstore
            3. Initializes the Annoy database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain import Annoy
                from langchain.embeddings import OpenAIEmbeddings


from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                index = Annoy.from_texts(texts, embeddings)
        """
        embeddings = embedding.embed_documents(texts)
        return cls.__from(
            texts, embeddings, embedding, metadatas, metric, trees, n_jobs, **kwargs
        )
[docs]    @classmethod
    def from_embeddings(
        cls,
        text_embeddings: List[Tuple[str, List[float]]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        """Construct Annoy wrapper from embeddings.
        Args:
            text_embeddings: List of tuples of (text, embedding)
            embedding: Embedding function to use.
            metadatas: List of metadata dictionaries to associate with documents.
            metric: Metric to use for indexing. Defaults to "angular".
            trees: Number of trees to use for indexing. Defaults to 100.
            n_jobs: Number of jobs to use for indexing. Defaults to -1
        This is a user friendly interface that:
            1. Creates an in memory docstore with provided embeddings
            2. Initializes the Annoy database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain import Annoy
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                text_embeddings = embeddings.embed_documents(texts)
                text_embedding_pairs = list(zip(texts, text_embeddings))


text_embedding_pairs = list(zip(texts, text_embeddings))
                db = Annoy.from_embeddings(text_embedding_pairs, embeddings)
        """
        texts = [t[0] for t in text_embeddings]
        embeddings = [t[1] for t in text_embeddings]
        return cls.__from(
            texts, embeddings, embedding, metadatas, metric, trees, n_jobs, **kwargs
        )
[docs]    def save_local(self, folder_path: str, prefault: bool = False) -> None:
        """Save Annoy index, docstore, and index_to_docstore_id to disk.
        Args:
            folder_path: folder path to save index, docstore,
                and index_to_docstore_id to.
            prefault: Whether to pre-load the index into memory.
        """
        path = Path(folder_path)
        os.makedirs(path, exist_ok=True)
        # save index, index config, docstore and index_to_docstore_id
        config_object = ConfigParser()
        config_object["ANNOY"] = {
            "f": self.index.f,
            "metric": self.metric,
        }
        self.index.save(str(path / "index.annoy"), prefault=prefault)
        with open(path / "index.pkl", "wb") as file:
            pickle.dump((self.docstore, self.index_to_docstore_id, config_object), file)
[docs]    @classmethod
    def load_local(
        cls,
        folder_path: str,
        embeddings: Embeddings,
    ) -> Annoy:
        """Load Annoy index, docstore, and index_to_docstore_id to disk.
        Args:
            folder_path: folder path to load index, docstore,


Args:
            folder_path: folder path to load index, docstore,
                and index_to_docstore_id from.
            embeddings: Embeddings to use when generating queries.
        """
        path = Path(folder_path)
        # load index separately since it is not picklable
        annoy = dependable_annoy_import()
        # load docstore and index_to_docstore_id
        with open(path / "index.pkl", "rb") as file:
            docstore, index_to_docstore_id, config_object = pickle.load(file)
        f = int(config_object["ANNOY"]["f"])
        metric = config_object["ANNOY"]["metric"]
        index = annoy.AnnoyIndex(f, metric=metric)
        index.load(str(path / "index.annoy"))
        return cls(
            embeddings.embed_query, index, metric, docstore, index_to_docstore_id
        )
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.faiss
"""Wrapper around FAISS vector database."""
from __future__ import annotations
import math
import os
import pickle
import uuid
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
import numpy as np
from langchain.docstore.base import AddableMixin, Docstore
from langchain.docstore.document import Document
from langchain.docstore.in_memory import InMemoryDocstore
from langchain.embeddings.base import Embeddings
from langchain.vectorstores.base import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
def dependable_faiss_import(no_avx2: Optional[bool] = None) -> Any:
    """
    Import faiss if available, otherwise raise error.
    If FAISS_NO_AVX2 environment variable is set, it will be considered
    to load FAISS with no AVX2 optimization.
    Args:
        no_avx2: Load FAISS strictly with no AVX2 optimization
            so that the vectorstore is portable and compatible with other devices.
    """
    if no_avx2 is None and "FAISS_NO_AVX2" in os.environ:
        no_avx2 = bool(os.getenv("FAISS_NO_AVX2"))
    try:
        if no_avx2:
            from faiss import swigfaiss as faiss
        else:
            import faiss
    except ImportError:
        raise ValueError(
            "Could not import faiss python package. "
            "Please install it with `pip install faiss` "
            "or `pip install faiss-cpu` (depending on Python version)."
        )
    return faiss
def _default_relevance_score_fn(score: float) -> float:


return faiss
def _default_relevance_score_fn(score: float) -> float:
    """Return a similarity score on a scale [0, 1]."""
    # The 'correct' relevance function
    # may differ depending on a few things, including:
    # - the distance / similarity metric used by the VectorStore
    # - the scale of your embeddings (OpenAI's are unit normed. Many others are not!)
    # - embedding dimensionality
    # - etc.
    # This function converts the euclidean norm of normalized embeddings
    # (0 is most similar, sqrt(2) most dissimilar)
    # to a similarity function (0 to 1)
    return 1.0 - score / math.sqrt(2)
[docs]class FAISS(VectorStore):
    """Wrapper around FAISS vector database.
    To use, you should have the ``faiss`` python package installed.
    Example:
        .. code-block:: python
            from langchain import FAISS
            faiss = FAISS(embedding_function, index, docstore, index_to_docstore_id)
    """
    def __init__(
        self,
        embedding_function: Callable,
        index: Any,
        docstore: Docstore,
        index_to_docstore_id: Dict[int, str],
        relevance_score_fn: Optional[
            Callable[[float], float]
        ] = _default_relevance_score_fn,
        normalize_L2: bool = False,
    ):
        """Initialize with necessary components."""
        self.embedding_function = embedding_function
        self.index = index
        self.docstore = docstore
        self.index_to_docstore_id = index_to_docstore_id
        self.relevance_score_fn = relevance_score_fn
        self._normalize_L2 = normalize_L2


self._normalize_L2 = normalize_L2
    def __add(
        self,
        texts: Iterable[str],
        embeddings: Iterable[List[float]],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        if not isinstance(self.docstore, AddableMixin):
            raise ValueError(
                "If trying to add texts, the underlying docstore should support "
                f"adding items, which {self.docstore} does not"
            )
        documents = []
        for i, text in enumerate(texts):
            metadata = metadatas[i] if metadatas else {}
            documents.append(Document(page_content=text, metadata=metadata))
        if ids is None:
            ids = [str(uuid.uuid4()) for _ in texts]
        # Add to the index, the index_to_id mapping, and the docstore.
        starting_len = len(self.index_to_docstore_id)
        faiss = dependable_faiss_import()
        vector = np.array(embeddings, dtype=np.float32)
        if self._normalize_L2:
            faiss.normalize_L2(vector)
        self.index.add(vector)
        # Get list of index, id, and docs.
        full_info = [(starting_len + i, ids[i], doc) for i, doc in enumerate(documents)]
        # Add information to docstore and index.
        self.docstore.add({_id: doc for _, _id, doc in full_info})
        index_to_id = {index: _id for index, _id, _ in full_info}
        self.index_to_docstore_id.update(index_to_id)
        return [_id for _, _id, _ in full_info]


return [_id for _, _id, _ in full_info]
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of unique IDs.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        if not isinstance(self.docstore, AddableMixin):
            raise ValueError(
                "If trying to add texts, the underlying docstore should support "
                f"adding items, which {self.docstore} does not"
            )
        # Embed and create the documents.
        embeddings = [self.embedding_function(text) for text in texts]
        return self.__add(texts, embeddings, metadatas=metadatas, ids=ids, **kwargs)
[docs]    def add_embeddings(
        self,
        text_embeddings: Iterable[Tuple[str, List[float]]],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Args:
            text_embeddings: Iterable pairs of string and embedding to
                add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of unique IDs.
        Returns:


ids: Optional list of unique IDs.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        if not isinstance(self.docstore, AddableMixin):
            raise ValueError(
                "If trying to add texts, the underlying docstore should support "
                f"adding items, which {self.docstore} does not"
            )
        # Embed and create the documents.
        texts, embeddings = zip(*text_embeddings)
        return self.__add(texts, embeddings, metadatas=metadatas, ids=ids, **kwargs)
[docs]    def similarity_search_with_score_by_vector(
        self, embedding: List[float], k: int = 4
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            embedding: Embedding vector to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of documents most similar to the query text and L2 distance
            in float for each. Lower score represents more similarity.
        """
        faiss = dependable_faiss_import()
        vector = np.array([embedding], dtype=np.float32)
        if self._normalize_L2:
            faiss.normalize_L2(vector)
        scores, indices = self.index.search(vector, k)
        docs = []
        for j, i in enumerate(indices[0]):
            if i == -1:
                # This happens when not enough docs are returned.
                continue
            _id = self.index_to_docstore_id[i]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")


raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append((doc, scores[0][j]))
        return docs
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of documents most similar to the query text with
            L2 distance in float. Lower score represents more similarity.
        """
        embedding = self.embedding_function(query)
        docs = self.similarity_search_with_score_by_vector(embedding, k)
        return docs
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to embedding vector.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_vector(embedding, k)
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:


k: Number of Documents to return. Defaults to 4.
        Returns:
            List of Documents most similar to the query.
        """
        docs_and_scores = self.similarity_search_with_score(query, k)
        return [doc for doc, _ in docs_and_scores]
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        _, indices = self.index.search(np.array([embedding], dtype=np.float32), fetch_k)
        # -1 happens when not enough docs are returned.
        embeddings = [self.index.reconstruct(int(i)) for i in indices[0] if i != -1]
        mmr_selected = maximal_marginal_relevance(
            np.array([embedding], dtype=np.float32),
            embeddings,
            k=k,
            lambda_mult=lambda_mult,
        )


embeddings,
            k=k,
            lambda_mult=lambda_mult,
        )
        selected_indices = [indices[0][i] for i in mmr_selected]
        docs = []
        for i in selected_indices:
            if i == -1:
                # This happens when not enough docs are returned.
                continue
            _id = self.index_to_docstore_id[i]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append(doc)
        return docs
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding = self.embedding_function(query)
        docs = self.max_marginal_relevance_search_by_vector(
            embedding, k, fetch_k, lambda_mult=lambda_mult
        )
        return docs


embedding, k, fetch_k, lambda_mult=lambda_mult
        )
        return docs
[docs]    def merge_from(self, target: FAISS) -> None:
        """Merge another FAISS object with the current one.
        Add the target FAISS to the current one.
        Args:
            target: FAISS object you wish to merge into the current one
        Returns:
            None.
        """
        if not isinstance(self.docstore, AddableMixin):
            raise ValueError("Cannot merge with this type of docstore")
        # Numerical index for target docs are incremental on existing ones
        starting_len = len(self.index_to_docstore_id)
        # Merge two IndexFlatL2
        self.index.merge_from(target.index)
        # Get id and docs from target FAISS object
        full_info = []
        for i, target_id in target.index_to_docstore_id.items():
            doc = target.docstore.search(target_id)
            if not isinstance(doc, Document):
                raise ValueError("Document should be returned")
            full_info.append((starting_len + i, target_id, doc))
        # Add information to docstore and index_to_docstore_id.
        self.docstore.add({_id: doc for _, _id, doc in full_info})
        index_to_id = {index: _id for index, _id, _ in full_info}
        self.index_to_docstore_id.update(index_to_id)
    @classmethod
    def __from(
        cls,
        texts: List[str],
        embeddings: List[List[float]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        normalize_L2: bool = False,
        **kwargs: Any,
    ) -> FAISS:


**kwargs: Any,
    ) -> FAISS:
        faiss = dependable_faiss_import()
        index = faiss.IndexFlatL2(len(embeddings[0]))
        vector = np.array(embeddings, dtype=np.float32)
        if normalize_L2:
            faiss.normalize_L2(vector)
        index.add(vector)
        documents = []
        if ids is None:
            ids = [str(uuid.uuid4()) for _ in texts]
        for i, text in enumerate(texts):
            metadata = metadatas[i] if metadatas else {}
            documents.append(Document(page_content=text, metadata=metadata))
        index_to_id = dict(enumerate(ids))
        docstore = InMemoryDocstore(dict(zip(index_to_id.values(), documents)))
        return cls(
            embedding.embed_query,
            index,
            docstore,
            index_to_id,
            normalize_L2=normalize_L2,
            **kwargs,
        )
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> FAISS:
        """Construct FAISS wrapper from raw documents.
        This is a user friendly interface that:
            1. Embeds documents.
            2. Creates an in memory docstore
            3. Initializes the FAISS database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain import FAISS
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()


embeddings = OpenAIEmbeddings()
                faiss = FAISS.from_texts(texts, embeddings)
        """
        embeddings = embedding.embed_documents(texts)
        return cls.__from(
            texts,
            embeddings,
            embedding,
            metadatas=metadatas,
            ids=ids,
            **kwargs,
        )
[docs]    @classmethod
    def from_embeddings(
        cls,
        text_embeddings: List[Tuple[str, List[float]]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> FAISS:
        """Construct FAISS wrapper from raw documents.
        This is a user friendly interface that:
            1. Embeds documents.
            2. Creates an in memory docstore
            3. Initializes the FAISS database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain import FAISS
                from langchain.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                text_embeddings = embeddings.embed_documents(texts)
                text_embedding_pairs = list(zip(texts, text_embeddings))
                faiss = FAISS.from_embeddings(text_embedding_pairs, embeddings)
        """
        texts = [t[0] for t in text_embeddings]
        embeddings = [t[1] for t in text_embeddings]
        return cls.__from(
            texts,
            embeddings,
            embedding,
            metadatas=metadatas,
            ids=ids,
            **kwargs,
        )


ids=ids,
            **kwargs,
        )
[docs]    def save_local(self, folder_path: str, index_name: str = "index") -> None:
        """Save FAISS index, docstore, and index_to_docstore_id to disk.
        Args:
            folder_path: folder path to save index, docstore,
                and index_to_docstore_id to.
            index_name: for saving with a specific index file name
        """
        path = Path(folder_path)
        path.mkdir(exist_ok=True, parents=True)
        # save index separately since it is not picklable
        faiss = dependable_faiss_import()
        faiss.write_index(
            self.index, str(path / "{index_name}.faiss".format(index_name=index_name))
        )
        # save docstore and index_to_docstore_id
        with open(path / "{index_name}.pkl".format(index_name=index_name), "wb") as f:
            pickle.dump((self.docstore, self.index_to_docstore_id), f)
[docs]    @classmethod
    def load_local(
        cls, folder_path: str, embeddings: Embeddings, index_name: str = "index"
    ) -> FAISS:
        """Load FAISS index, docstore, and index_to_docstore_id from disk.
        Args:
            folder_path: folder path to load index, docstore,
                and index_to_docstore_id from.
            embeddings: Embeddings to use when generating queries
            index_name: for saving with a specific index file name
        """
        path = Path(folder_path)
        # load index separately since it is not picklable
        faiss = dependable_faiss_import()
        index = faiss.read_index(


faiss = dependable_faiss_import()
        index = faiss.read_index(
            str(path / "{index_name}.faiss".format(index_name=index_name))
        )
        # load docstore and index_to_docstore_id
        with open(path / "{index_name}.pkl".format(index_name=index_name), "rb") as f:
            docstore, index_to_docstore_id = pickle.load(f)
        return cls(embeddings.embed_query, index, docstore, index_to_docstore_id)
    def _similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return docs and their similarity scores on a scale from 0 to 1."""
        if self.relevance_score_fn is None:
            raise ValueError(
                "normalize_score_fn must be provided to"
                " FAISS constructor to normalize scores"
            )
        docs_and_scores = self.similarity_search_with_score(query, k=k)
        return [(doc, self.relevance_score_fn(score)) for doc, score in docs_and_scores]
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.typesense
"""Wrapper around Typesense vector search"""
from __future__ import annotations
import uuid
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union
from langchain.docstore.document import Document
from langchain.embeddings.base import Embeddings
from langchain.utils import get_from_env
from langchain.vectorstores.base import VectorStore
if TYPE_CHECKING:
    from typesense.client import Client
    from typesense.collection import Collection
[docs]class Typesense(VectorStore):
    """Wrapper around Typesense vector search.
    To use, you should have the ``typesense`` python package installed.
    Example:
        .. code-block:: python
            from langchain.embedding.openai import OpenAIEmbeddings
            from langchain.vectorstores import Typesense
            import typesense
            node = {
                "host": "localhost",  # For Typesense Cloud use xxx.a1.typesense.net
                "port": "8108",       # For Typesense Cloud use 443
                "protocol": "http"    # For Typesense Cloud use https
            }
            typesense_client = typesense.Client(
                {
                  "nodes": [node],
                  "api_key": "<API_KEY>",
                  "connection_timeout_seconds": 2
                }
            )
            typesense_collection_name = "langchain-memory"
            embedding = OpenAIEmbeddings()
            vectorstore = Typesense(
                typesense_client,
                typesense_collection_name,
                embedding.embed_query,
                "text",
            )
    """
    def __init__(
        self,
        typesense_client: Client,
        embedding: Embeddings,
        *,
        typesense_collection_name: Optional[str] = None,


*,
        typesense_collection_name: Optional[str] = None,
        text_key: str = "text",
    ):
        """Initialize with Typesense client."""
        try:
            from typesense import Client
        except ImportError:
            raise ValueError(
                "Could not import typesense python package. "
                "Please install it with `pip install typesense`."
            )
        if not isinstance(typesense_client, Client):
            raise ValueError(
                f"typesense_client should be an instance of typesense.Client, "
                f"got {type(typesense_client)}"
            )
        self._typesense_client = typesense_client
        self._embedding = embedding
        self._typesense_collection_name = (
            typesense_collection_name or f"langchain-{str(uuid.uuid4())}"
        )
        self._text_key = text_key
    @property
    def _collection(self) -> Collection:
        return self._typesense_client.collections[self._typesense_collection_name]
    def _prep_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]],
        ids: Optional[List[str]],
    ) -> List[dict]:
        """Embed and create the documents"""
        _ids = ids or (str(uuid.uuid4()) for _ in texts)
        _metadatas: Iterable[dict] = metadatas or ({} for _ in texts)
        embedded_texts = self._embedding.embed_documents(list(texts))
        return [
            {"id": _id, "vec": vec, f"{self._text_key}": text, "metadata": metadata}
            for _id, vec, text, metadata in zip(_ids, embedded_texts, texts, _metadatas)
        ]


]
    def _create_collection(self, num_dim: int) -> None:
        fields = [
            {"name": "vec", "type": "float[]", "num_dim": num_dim},
            {"name": f"{self._text_key}", "type": "string"},
            {"name": ".*", "type": "auto"},
        ]
        self._typesense_client.collections.create(
            {"name": self._typesense_collection_name, "fields": fields}
        )
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embedding and add to the vectorstore.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of ids to associate with the texts.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        from typesense.exceptions import ObjectNotFound
        docs = self._prep_texts(texts, metadatas, ids)
        try:
            self._collection.documents.import_(docs, {"action": "upsert"})
        except ObjectNotFound:
            # Create the collection if it doesn't already exist
            self._create_collection(len(docs[0]["vec"]))
            self._collection.documents.import_(docs, {"action": "upsert"})
        return [doc["id"] for doc in docs]
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,


self,
        query: str,
        k: int = 4,
        filter: Optional[str] = "",
    ) -> List[Tuple[Document, float]]:
        """Return typesense documents most similar to query, along with scores.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            filter: typesense filter_by expression to filter documents on
        Returns:
            List of Documents most similar to the query and score for each
        """
        embedded_query = [str(x) for x in self._embedding.embed_query(query)]
        query_obj = {
            "q": "*",
            "vector_query": f'vec:([{",".join(embedded_query)}], k:{k})',
            "filter_by": filter,
            "collection": self._typesense_collection_name,
        }
        docs = []
        response = self._typesense_client.multi_search.perform(
            {"searches": [query_obj]}, {}
        )
        for hit in response["results"][0]["hits"]:
            document = hit["document"]
            metadata = document["metadata"]
            text = document[self._text_key]
            score = hit["vector_distance"]
            docs.append((Document(page_content=text, metadata=metadata), score))
        return docs
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        filter: Optional[str] = "",
        **kwargs: Any,
    ) -> List[Document]:
        """Return typesense documents most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.


k: Number of Documents to return. Defaults to 4.
            filter: typesense filter_by expression to filter documents on
        Returns:
            List of Documents most similar to the query and score for each
        """
        docs_and_score = self.similarity_search_with_score(query, k=k, filter=filter)
        return [doc for doc, _ in docs_and_score]
[docs]    @classmethod
    def from_client_params(
        cls,
        embedding: Embeddings,
        *,
        host: str = "localhost",
        port: Union[str, int] = "8108",
        protocol: str = "http",
        typesense_api_key: Optional[str] = None,
        connection_timeout_seconds: int = 2,
        **kwargs: Any,
    ) -> Typesense:
        """Initialize Typesense directly from client parameters.
        Example:
            .. code-block:: python
                from langchain.embedding.openai import OpenAIEmbeddings
                from langchain.vectorstores import Typesense
                # Pass in typesense_api_key as kwarg or set env var "TYPESENSE_API_KEY".
                vectorstore = Typesense(
                    OpenAIEmbeddings(),
                    host="localhost",
                    port="8108",
                    protocol="http",
                    typesense_collection_name="langchain-memory",
                )
        """
        try:
            from typesense import Client
        except ImportError:
            raise ValueError(
                "Could not import typesense python package. "
                "Please install it with `pip install typesense`."
            )
        node = {
            "host": host,
            "port": str(port),
            "protocol": protocol,
        }
        typesense_api_key = typesense_api_key or get_from_env(


}
        typesense_api_key = typesense_api_key or get_from_env(
            "typesense_api_key", "TYPESENSE_API_KEY"
        )
        client_config = {
            "nodes": [node],
            "api_key": typesense_api_key,
            "connection_timeout_seconds": connection_timeout_seconds,
        }
        return cls(Client(client_config), embedding, **kwargs)
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        typesense_client: Optional[Client] = None,
        typesense_client_params: Optional[dict] = None,
        typesense_collection_name: Optional[str] = None,
        text_key: str = "text",
        **kwargs: Any,
    ) -> Typesense:
        """Construct Typesense wrapper from raw text."""
        if typesense_client:
            vectorstore = cls(typesense_client, embedding, **kwargs)
        elif typesense_client_params:
            vectorstore = cls.from_client_params(
                embedding, **typesense_client_params, **kwargs
            )
        else:
            raise ValueError(
                "Must specify one of typesense_client or typesense_client_params."
            )
        vectorstore.add_texts(texts, metadatas=metadatas, ids=ids)
        return vectorstore
By Harrison Chase

      © Copyright 2023, Harrison Chase.

  Last updated on Jun 07, 2023.


Source code for langchain.vectorstores.supabase
from __future__ import annotations
from itertools import repeat
from typing import (
    TYPE_CHECKING,
    Any,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
    Union,
)
import numpy as np
from langchain.docstore.document import Document
from langchain.embeddings.base import Embeddings
from langchain.vectorstores.base import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
if TYPE_CHECKING:
    import supabase
[docs]class SupabaseVectorStore(VectorStore):
    """VectorStore for a Supabase postgres database. Assumes you have the `pgvector`
    extension installed and a `match_documents` (or similar) function. For more details:
    https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase
    You can implement your own `match_documents` function in order to limit the search
    space to a subset of documents based on your own authorization or business logic.
    Note that the Supabase Python client does not yet support async operations.
    If you'd like to use `max_marginal_relevance_search`, please review the instructions
    below on modifying the `match_documents` function to return matched embeddings.
    """
    _client: supabase.client.Client
    # This is the embedding function. Don't confuse with the embedding vectors.
    # We should perhaps rename the underlying Embedding base class to EmbeddingFunction
    # or something
    _embedding: Embeddings
    table_name: str
    query_name: str
    def __init__(
        self,
        client: supabase.client.Client,
        embedding: Embeddings,
        table_name: str,


embedding: Embeddings,
        table_name: str,
        query_name: Union[str, None] = None,
    ) -> None:
        """Initialize with supabase client."""
        try:
            import supabase  # noqa: F401
        except ImportError:
            raise ValueError(
                "Could not import supabase python package. "
                "Please install it with `pip install supabase`."
            )
        self._client = client
        self._embedding: Embeddings = embedding
        self.table_name = table_name or "documents"
        self.query_name = query_name or "match_documents"
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict[Any, Any]]] = None,
        **kwargs: Any,
    ) -> List[str]:
        docs = self._texts_to_documents(texts, metadatas)
        vectors = self._embedding.embed_documents(list(texts))
        return self.add_vectors(vectors, docs)
[docs]    @classmethod
    def from_texts(
        cls: Type["SupabaseVectorStore"],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        client: Optional[supabase.client.Client] = None,
        table_name: Optional[str] = "documents",
        query_name: Union[str, None] = "match_documents",
        **kwargs: Any,
    ) -> "SupabaseVectorStore":
        """Return VectorStore initialized from texts and embeddings."""
        if not client:
            raise ValueError("Supabase client is required.")
        if not table_name:
            raise ValueError("Supabase document table_name is required.")