viruPVS/langchain-docs huggingface dataset:
)
        # List of arguments that needs to be removed from kwargs
        # before passing kwargs to get opensearch client
        keys_list = [
            "opensearch_url",
            "index_name",
            "is_appx_search",
            "vector_field",
            "text_field",
            "engine",
            "space_type",
            "ef_search",
            "ef_construction",
            "m",
            "max_chunk_bytes",
            "is_aoss",
        ]
        _validate_embeddings_and_bulk_size(len(embeddings), bulk_size)
        dim = len(embeddings[0])
        # Get the index name from either from kwargs or ENV Variable
        # before falling back to random generation
        index_name = get_from_dict_or_env(
            kwargs, "index_name", "OPENSEARCH_INDEX_NAME", default=uuid.uuid4().hex
        )
        is_appx_search = kwargs.get("is_appx_search", True)
        vector_field = kwargs.get("vector_field", "vector_field")
        text_field = kwargs.get("text_field", "text")
        max_chunk_bytes = kwargs.get("max_chunk_bytes", 1 * 1024 * 1024)
        http_auth = kwargs.get("http_auth")
        is_aoss = _is_aoss_enabled(http_auth=http_auth)
        engine = None
        if is_aoss and not is_appx_search:
            raise ValueError(
                "Amazon OpenSearch Service Serverless only "
                "supports `approximate_search`"
            )
        if is_appx_search:
            engine = kwargs.get("engine", "nmslib")
            space_type = kwargs.get("space_type", "l2")
            ef_search = kwargs.get("ef_search", 512)
ef_search = kwargs.get("ef_search", 512)
            ef_construction = kwargs.get("ef_construction", 512)
            m = kwargs.get("m", 16)
            _validate_aoss_with_engines(is_aoss, engine)
            mapping = _default_text_mapping(
                dim, engine, space_type, ef_search, ef_construction, m, vector_field
            )
        else:
            mapping = _default_scripting_text_mapping(dim)
        [kwargs.pop(key, None) for key in keys_list]
        client = _get_opensearch_client(opensearch_url, **kwargs)
        _bulk_ingest_embeddings(
            client,
            index_name,
            embeddings,
            texts,
            ids=ids,
            metadatas=metadatas,
            vector_field=vector_field,
            text_field=text_field,
            mapping=mapping,
            max_chunk_bytes=max_chunk_bytes,
            is_aoss=is_aoss,
        )
        kwargs["engine"] = engine
        return cls(opensearch_url, index_name, embedding, **kwargs)
Source code for langchain_community.vectorstores.meilisearch
from __future__ import annotations
import uuid
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Type
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.utils import get_from_env
from langchain_core.vectorstores import VectorStore
if TYPE_CHECKING:
    from meilisearch import Client
def _create_client(
    client: Optional[Client] = None,
    url: Optional[str] = None,
    api_key: Optional[str] = None,
) -> Client:
    try:
        import meilisearch
    except ImportError:
        raise ImportError(
            "Could not import meilisearch python package. "
            "Please install it with `pip install meilisearch`."
        )
    if not client:
        url = url or get_from_env("url", "MEILI_HTTP_ADDR")
        try:
            api_key = api_key or get_from_env("api_key", "MEILI_MASTER_KEY")
        except Exception:
            pass
        client = meilisearch.Client(url=url, api_key=api_key)
    elif not isinstance(client, meilisearch.Client):
        raise ValueError(
            f"client should be an instance of meilisearch.Client, "
            f"got {type(client)}"
        )
    try:
        client.version()
    except ValueError as e:
        raise ValueError(f"Failed to connect to Meilisearch: {e}")
    return client
[docs]class Meilisearch(VectorStore):
    """`Meilisearch` vector store.
    To use this, you need to have `meilisearch` python package installed,
To use this, you need to have `meilisearch` python package installed,
    and a running Meilisearch instance.
    To learn more about Meilisearch Python, refer to the in-depth
    Meilisearch Python documentation: https://meilisearch.github.io/meilisearch-python/.
    See the following documentation for how to run a Meilisearch instance:
    https://www.meilisearch.com/docs/learn/getting_started/quick_start.
    Example:
        .. code-block:: python
            from langchain_community.vectorstores import Meilisearch
            from langchain_community.embeddings.openai import OpenAIEmbeddings
            import meilisearch
            # api_key is optional; provide it if your meilisearch instance requires it
            client = meilisearch.Client(url='http://127.0.0.1:7700', api_key='***')
            embeddings = OpenAIEmbeddings()
            vectorstore = Meilisearch(
                embedding=embeddings,
                client=client,
                index_name='langchain_demo',
                text_key='text')
    """
[docs]    def __init__(
        self,
        embedding: Embeddings,
        client: Optional[Client] = None,
        url: Optional[str] = None,
        api_key: Optional[str] = None,
        index_name: str = "langchain-demo",
        text_key: str = "text",
        metadata_key: str = "metadata",
    ):
        """Initialize with Meilisearch client."""
        client = _create_client(client=client, url=url, api_key=api_key)
        self._client = client
        self._index_name = index_name
        self._embedding = embedding
self._index_name = index_name
        self._embedding = embedding
        self._text_key = text_key
        self._metadata_key = metadata_key
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embedding and add them to the vector store.
        Args:
            texts (Iterable[str]): Iterable of strings/text to add to the vectorstore.
            metadatas (Optional[List[dict]]): Optional list of metadata.
                Defaults to None.
            ids Optional[List[str]]: Optional list of IDs.
                Defaults to None.
        Returns:
            List[str]: List of IDs of the texts added to the vectorstore.
        """
        texts = list(texts)
        # Embed and create the documents
        docs = []
        if ids is None:
            ids = [uuid.uuid4().hex for _ in texts]
        if metadatas is None:
            metadatas = [{} for _ in texts]
        embedding_vectors = self._embedding.embed_documents(texts)
        for i, text in enumerate(texts):
            id = ids[i]
            metadata = metadatas[i]
            metadata[self._text_key] = text
            embedding = embedding_vectors[i]
            docs.append(
                {
                    "id": id,
                    "_vectors": embedding,
                    f"{self._metadata_key}": metadata,
                }
            )
        # Send to Meilisearch
        self._client.index(str(self._index_name)).add_documents(docs)
        return ids
[docs]    def similarity_search(
return ids
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return meilisearch documents most similar to the query.
        Args:
            query (str): Query text for which to find similar documents.
            k (int): Number of documents to return. Defaults to 4.
            filter (Optional[Dict[str, str]]): Filter by metadata.
                Defaults to None.
        Returns:
            List[Document]: List of Documents most similar to the query
            text and score for each.
        """
        docs_and_scores = self.similarity_search_with_score(
            query=query,
            k=k,
            filter=filter,
            kwargs=kwargs,
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return meilisearch documents most similar to the query, along with scores.
        Args:
            query (str): Query text for which to find similar documents.
            k (int): Number of documents to return. Defaults to 4.
            filter (Optional[Dict[str, str]]): Filter by metadata.
                Defaults to None.
        Returns:
            List[Document]: List of Documents most similar to the query
            text and score for each.
        """
        _query = self._embedding.embed_query(query)
"""
        _query = self._embedding.embed_query(query)
        docs = self.similarity_search_by_vector_with_scores(
            embedding=_query,
            k=k,
            filter=filter,
            kwargs=kwargs,
        )
        return docs
[docs]    def similarity_search_by_vector_with_scores(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[Dict[str, Any]] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return meilisearch documents most similar to embedding vector.
        Args:
            embedding (List[float]): Embedding to look up similar documents.
            k (int): Number of documents to return. Defaults to 4.
            filter (Optional[Dict[str, str]]): Filter by metadata.
                Defaults to None.
        Returns:
            List[Document]: List of Documents most similar to the query
                vector and score for each.
        """
        docs = []
        results = self._client.index(str(self._index_name)).search(
            "", {"vector": embedding, "limit": k, "filter": filter}
        )
        for result in results["hits"]:
            metadata = result[self._metadata_key]
            if self._text_key in metadata:
                text = metadata.pop(self._text_key)
                semantic_score = result["_semanticScore"]
                docs.append(
                    (Document(page_content=text, metadata=metadata), semantic_score)
                )
        return docs
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
**kwargs: Any,
    ) -> List[Document]:
        """Return meilisearch documents most similar to embedding vector.
        Args:
            embedding (List[float]): Embedding to look up similar documents.
            k (int): Number of documents to return. Defaults to 4.
            filter (Optional[Dict[str, str]]): Filter by metadata.
                Defaults to None.
        Returns:
            List[Document]: List of Documents most similar to the query
                vector and score for each.
        """
        docs = self.similarity_search_by_vector_with_scores(
            embedding=embedding,
            k=k,
            filter=filter,
            kwargs=kwargs,
        )
        return [doc for doc, _ in docs]
[docs]    @classmethod
    def from_texts(
        cls: Type[Meilisearch],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        client: Optional[Client] = None,
        url: Optional[str] = None,
        api_key: Optional[str] = None,
        index_name: str = "langchain-demo",
        ids: Optional[List[str]] = None,
        text_key: Optional[str] = "text",
        metadata_key: Optional[str] = "metadata",
        **kwargs: Any,
    ) -> Meilisearch:
        """Construct Meilisearch wrapper from raw documents.
        This is a user-friendly interface that:
            1. Embeds documents.
            2. Adds the documents to a provided Meilisearch index.
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
Example:
            .. code-block:: python
                from langchain_community.vectorstores import Meilisearch
                from langchain_community.embeddings import OpenAIEmbeddings
                import meilisearch
                # The environment should be the one specified next to the API key
                # in your Meilisearch console
                client = meilisearch.Client(url='http://127.0.0.1:7700', api_key='***')
                embeddings = OpenAIEmbeddings()
                docsearch = Meilisearch.from_texts(
                    client=client,
                    embeddings=embeddings,
                )
        """
        client = _create_client(client=client, url=url, api_key=api_key)
        vectorstore = cls(
            embedding=embedding,
            client=client,
            index_name=index_name,
        )
        vectorstore.add_texts(
            texts=texts,
            metadatas=metadatas,
            ids=ids,
            text_key=text_key,
            metadata_key=metadata_key,
        )
        return vectorstore
Source code for langchain_community.vectorstores.astradb
from __future__ import annotations
import uuid
import warnings
from concurrent.futures import ThreadPoolExecutor
from typing import (
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Optional,
    Set,
    Tuple,
    Type,
    TypeVar,
)
import numpy as np
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.utils.iter import batch_iterate
from langchain_core.vectorstores import VectorStore
from langchain_community.vectorstores.utils import maximal_marginal_relevance
ADBVST = TypeVar("ADBVST", bound="AstraDB")
T = TypeVar("T")
U = TypeVar("U")
DocDict = Dict[str, Any]  # dicts expressing entries to insert
# Batch/concurrency default values (if parameters not provided):
# Size of batches for bulk insertions:
#   (20 is the max batch size for the HTTP API at the time of writing)
DEFAULT_BATCH_SIZE = 20
# Number of threads to insert batches concurrently:
DEFAULT_BULK_INSERT_BATCH_CONCURRENCY = 16
# Number of threads in a batch to insert pre-existing entries:
DEFAULT_BULK_INSERT_OVERWRITE_CONCURRENCY = 10
# Number of threads (for deleting multiple rows concurrently):
DEFAULT_BULK_DELETE_CONCURRENCY = 20
def _unique_list(lst: List[T], key: Callable[[T], U]) -> List[T]:
    visited_keys: Set[U] = set()
    new_lst = []
    for item in lst:
        item_key = key(item)
        if item_key not in visited_keys:
            visited_keys.add(item_key)
            new_lst.append(item)
    return new_lst
visited_keys.add(item_key)
            new_lst.append(item)
    return new_lst
[docs]class AstraDB(VectorStore):
    """Wrapper around DataStax Astra DB for vector-store workloads.
    To use it, you need a recent installation of the `astrapy` library
    and an Astra DB cloud database.
    For quickstart and details, visit:
        docs.datastax.com/en/astra/home/astra.html
    Example:
        .. code-block:: python
                from langchain_community.vectorstores import AstraDB
                from langchain_community.embeddings.openai import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                vectorstore = AstraDB(
                  embedding=embeddings,
                  collection_name="my_store",
                  token="AstraCS:...",
                  api_endpoint="https://<DB-ID>-us-east1.apps.astra.datastax.com"
                )
                vectorstore.add_texts(["Giraffes", "All good here"])
                results = vectorstore.similarity_search("Everything's ok", k=1)
      Constructor Args (only keyword-arguments accepted):
          embedding (Embeddings): embedding function to use.
          collection_name (str): name of the Astra DB collection to create/use.
          token (Optional[str]): API token for Astra DB usage.
          api_endpoint (Optional[str]): full URL to the API endpoint,
              such as "https://<DB-ID>-us-east1.apps.astra.datastax.com".
          astra_db_client (Optional[Any]): *alternative to token+api_endpoint*,
              you can pass an already-created 'astrapy.db.AstraDB' instance.
          namespace (Optional[str]): namespace (aka keyspace) where the
namespace (Optional[str]): namespace (aka keyspace) where the
              collection is created. Defaults to the database's "default namespace".
          metric (Optional[str]): similarity function to use out of those
              available in Astra DB. If left out, it will use Astra DB API's
              defaults (i.e. "cosine" - but, for performance reasons,
              "dot_product" is suggested if embeddings are normalized to one).
      Advanced arguments (coming with sensible defaults):
          batch_size (Optional[int]): Size of batches for bulk insertions.
          bulk_insert_batch_concurrency (Optional[int]): Number of threads
              to insert batches concurrently.
          bulk_insert_overwrite_concurrency (Optional[int]): Number of
              threads in a batch to insert pre-existing entries.
          bulk_delete_concurrency (Optional[int]): Number of threads
              (for deleting multiple rows concurrently).
          pre_delete_collection (Optional[bool]): whether to delete the collection
              before creating it. If False and the collection already exists,
              the collection will be used as is.
      A note on concurrency: as a rule of thumb, on a typical client machine
      it is suggested to keep the quantity
          bulk_insert_batch_concurrency * bulk_insert_overwrite_concurrency
      much below 1000 to avoid exhausting the client multithreading/networking
      resources. The hardcoded defaults are somewhat conservative to meet
      most machines' specs, but a sensible choice to test may be:
          bulk_insert_batch_concurrency = 80
          bulk_insert_overwrite_concurrency = 10
      A bit of experimentation is required to nail the best results here,
      depending on both the machine/network specs and the expected workload
      (specifically, how often a write is an update of an existing id).
      Remember you can pass concurrency settings to individual calls to
Remember you can pass concurrency settings to individual calls to
      add_texts and add_documents as well.
    """
    @staticmethod
    def _filter_to_metadata(filter_dict: Optional[Dict[str, str]]) -> Dict[str, Any]:
        if filter_dict is None:
            return {}
        else:
            return {f"metadata.{mdk}": mdv for mdk, mdv in filter_dict.items()}
[docs]    def __init__(
        self,
        *,
        embedding: Embeddings,
        collection_name: str,
        token: Optional[str] = None,
        api_endpoint: Optional[str] = None,
        astra_db_client: Optional[Any] = None,  # 'astrapy.db.AstraDB' if passed
        namespace: Optional[str] = None,
        metric: Optional[str] = None,
        batch_size: Optional[int] = None,
        bulk_insert_batch_concurrency: Optional[int] = None,
        bulk_insert_overwrite_concurrency: Optional[int] = None,
        bulk_delete_concurrency: Optional[int] = None,
        pre_delete_collection: bool = False,
    ) -> None:
        """
        Create an AstraDB vector store object. See class docstring for help.
        """
        try:
            from astrapy.db import (
                AstraDB as LibAstraDB,
            )
            from astrapy.db import (
                AstraDBCollection as LibAstraDBCollection,
            )
        except (ImportError, ModuleNotFoundError):
            raise ImportError(
                "Could not import a recent astrapy python package. "
                "Please install it with `pip install --upgrade astrapy`."
            )
        # Conflicting-arg checks:
)
        # Conflicting-arg checks:
        if astra_db_client is not None:
            if token is not None or api_endpoint is not None:
                raise ValueError(
                    "You cannot pass 'astra_db_client' to AstraDB if passing "
                    "'token' and 'api_endpoint'."
                )
        self.embedding = embedding
        self.collection_name = collection_name
        self.token = token
        self.api_endpoint = api_endpoint
        self.namespace = namespace
        # Concurrency settings
        self.batch_size: int = batch_size or DEFAULT_BATCH_SIZE
        self.bulk_insert_batch_concurrency: int = (
            bulk_insert_batch_concurrency or DEFAULT_BULK_INSERT_BATCH_CONCURRENCY
        )
        self.bulk_insert_overwrite_concurrency: int = (
            bulk_insert_overwrite_concurrency
            or DEFAULT_BULK_INSERT_OVERWRITE_CONCURRENCY
        )
        self.bulk_delete_concurrency: int = (
            bulk_delete_concurrency or DEFAULT_BULK_DELETE_CONCURRENCY
        )
        # "vector-related" settings
        self._embedding_dimension: Optional[int] = None
        self.metric = metric
        if astra_db_client is not None:
            self.astra_db = astra_db_client
        else:
            self.astra_db = LibAstraDB(
                token=self.token,
                api_endpoint=self.api_endpoint,
                namespace=self.namespace,
            )
        if not pre_delete_collection:
            self._provision_collection()
        else:
            self.clear()
        self.collection = LibAstraDBCollection(
            collection_name=self.collection_name,
            astra_db=self.astra_db,
        )
    def _get_embedding_dimension(self) -> int:
        if self._embedding_dimension is None:
            self._embedding_dimension = len(
if self._embedding_dimension is None:
            self._embedding_dimension = len(
                self.embedding.embed_query("This is a sample sentence.")
            )
        return self._embedding_dimension
    def _drop_collection(self) -> None:
        """
        Drop the collection from storage.
        This is meant as an internal-usage method, no members
        are set other than actual deletion on the backend.
        """
        _ = self.astra_db.delete_collection(
            collection_name=self.collection_name,
        )
        return None
    def _provision_collection(self) -> None:
        """
        Run the API invocation to create the collection on the backend.
        Internal-usage method, no object members are set,
        other than working on the underlying actual storage.
        """
        _ = self.astra_db.create_collection(
            dimension=self._get_embedding_dimension(),
            collection_name=self.collection_name,
            metric=self.metric,
        )
        return None
    @property
    def embeddings(self) -> Embeddings:
        return self.embedding
    @staticmethod
    def _dont_flip_the_cos_score(similarity0to1: float) -> float:
        """Keep similarity from client unchanged ad it's in [0:1] already."""
        return similarity0to1
    def _select_relevance_score_fn(self) -> Callable[[float], float]:
        """
        The underlying API calls already returns a "score proper",
        i.e. one in [0, 1] where higher means more *similar*,
        so here the final score transformation is not reversing the interval:
        """
        return self._dont_flip_the_cos_score
[docs]    def clear(self) -> None:
        """Empty the collection of all its stored entries."""
        self._drop_collection()