viruPVS/langchain-docs huggingface dataset:
to add, defaults to 2
        Returns:
            List of Documents most similar to the query and score for each.
        """
        data = json.dumps(
            {
                "query": [
                    {
                        "query": query,
                        "start": 0,
                        "num_results": k,
                        "context_config": {
                            "sentences_before": n_sentence_context,
                            "sentences_after": n_sentence_context,
                        },
                        "corpus_key": [
                            {
                                "customer_id": self._vectara_customer_id,
                                "corpus_id": self._vectara_corpus_id,
                                "metadataFilter": filter,
                                "lexical_interpolation_config": {"lambda": lambda_val},
                            }
                        ],
                    }
                ]
            }
        )
        response = self._session.post(
            headers=self._get_post_headers(),
            url="https://api.vectara.io/v1/query",
            data=data,
            timeout=self.vectara_api_timeout,
        )
        if response.status_code != 200:
            logger.error(
                "Query failed %s",
                f"(code {response.status_code}, reason {response.reason}, details "
                f"{response.text})",
            )
            return []
        result = response.json()
        if score_threshold:
            responses = [
                r
                for r in result["responseSet"][0]["response"]
                if r["score"] > score_threshold
            ]
        else:
            responses = result["responseSet"][0]["response"]
        documents = result["responseSet"][0]["document"]
        metadatas = []
        for x in responses:
            md = {m["name"]: m["value"] for m in x["metadata"]}
md = {m["name"]: m["value"] for m in x["metadata"]}
            doc_num = x["documentIndex"]
            doc_md = {m["name"]: m["value"] for m in documents[doc_num]["metadata"]}
            md.update(doc_md)
            metadatas.append(md)
        docs_with_score = [
            (
                Document(
                    page_content=x["text"],
                    metadata=md,
                ),
                x["score"],
            )
            for x, md in zip(responses, metadatas)
        ]
        return docs_with_score
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 5,
        lambda_val: float = 0.025,
        filter: Optional[str] = None,
        n_sentence_context: int = 2,
        **kwargs: Any,
    ) -> List[Document]:
        """Return Vectara documents most similar to query, along with scores.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 5.
            filter: Dictionary of argument(s) to filter on metadata. For example a
                filter can be "doc.rating > 3.0 and part.lang = 'deu'"} see
                https://docs.vectara.com/docs/search-apis/sql/filter-overview for more
                details.
            n_sentence_context: number of sentences before/after the matching segment
                to add, defaults to 2
        Returns:
            List of Documents most similar to the query
        """
        docs_and_scores = self.similarity_search_with_score(
            query,
            k=k,
            lambda_val=lambda_val,
            filter=filter,
k=k,
            lambda_val=lambda_val,
            filter=filter,
            score_threshold=None,
            n_sentence_context=n_sentence_context,
            **kwargs,
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    @classmethod
    def from_texts(
        cls: Type[Vectara],
        texts: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> Vectara:
        """Construct Vectara wrapper from raw documents.
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain_community.vectorstores import Vectara
                vectara = Vectara.from_texts(
                    texts,
                    vectara_customer_id=customer_id,
                    vectara_corpus_id=corpus_id,
                    vectara_api_key=api_key,
                )
        """
        # Notes:
        # * Vectara generates its own embeddings, so we ignore the provided
        #   embeddings (required by interface)
        # * when metadatas[] are provided they are associated with each "part"
        #   in Vectara. doc_metadata can be used to provide additional metadata
        #   for the document itself (applies to all "texts" in this call)
        doc_metadata = kwargs.pop("doc_metadata", {})
        vectara = cls(**kwargs)
        vectara.add_texts(texts, metadatas, doc_metadata=doc_metadata, **kwargs)
        return vectara
[docs]    @classmethod
    def from_files(
        cls: Type[Vectara],
        files: List[str],
        embedding: Optional[Embeddings] = None,
files: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> Vectara:
        """Construct Vectara wrapper from raw documents.
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain_community.vectorstores import Vectara
                vectara = Vectara.from_files(
                    files_list,
                    vectara_customer_id=customer_id,
                    vectara_corpus_id=corpus_id,
                    vectara_api_key=api_key,
                )
        """
        # Note: Vectara generates its own embeddings, so we ignore the provided
        # embeddings (required by interface)
        vectara = cls(**kwargs)
        vectara.add_files(files, metadatas)
        return vectara
[docs]    def as_retriever(self, **kwargs: Any) -> VectaraRetriever:
        tags = kwargs.pop("tags", None) or []
        tags.extend(self._get_retriever_tags())
        return VectaraRetriever(vectorstore=self, search_kwargs=kwargs, tags=tags)
[docs]class VectaraRetriever(VectorStoreRetriever):
    """Retriever class for `Vectara`."""
    vectorstore: Vectara
    """Vectara vectorstore."""
    search_kwargs: dict = Field(
        default_factory=lambda: {
            "lambda_val": 0.0,
            "k": 5,
            "filter": "",
            "n_sentence_context": "2",
        }
    )
    """Search params.
        k: Number of Documents to return. Defaults to 5.
k: Number of Documents to return. Defaults to 5.
        lambda_val: lexical match parameter for hybrid search.
        filter: Dictionary of argument(s) to filter on metadata. For example a
            filter can be "doc.rating > 3.0 and part.lang = 'deu'"} see
            https://docs.vectara.com/docs/search-apis/sql/filter-overview
            for more details.
        n_sentence_context: number of sentences before/after the matching segment to add
    """
[docs]    def add_texts(
        self,
        texts: List[str],
        metadatas: Optional[List[dict]] = None,
        doc_metadata: Optional[dict] = None,
    ) -> None:
        """Add text to the Vectara vectorstore.
        Args:
            texts (List[str]): The text
            metadatas (List[dict]): Metadata dicts, must line up with existing store
        """
        self.vectorstore.add_texts(texts, metadatas, doc_metadata or {})
Source code for langchain_community.vectorstores.bageldb
from __future__ import annotations
import uuid
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
)
if TYPE_CHECKING:
    import bagel
    import bagel.config
    from bagel.api.types import ID, OneOrMany, Where, WhereDocument
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.utils import xor_args
from langchain_core.vectorstores import VectorStore
DEFAULT_K = 5
def _results_to_docs(results: Any) -> List[Document]:
    return [doc for doc, _ in _results_to_docs_and_scores(results)]
def _results_to_docs_and_scores(results: Any) -> List[Tuple[Document, float]]:
    return [
        (Document(page_content=result[0], metadata=result[1] or {}), result[2])
        for result in zip(
            results["documents"][0],
            results["metadatas"][0],
            results["distances"][0],
        )
    ]
[docs]class Bagel(VectorStore):
    """``BagelDB.ai`` vector store.
    To use, you should have the ``betabageldb`` python package installed.
    Example:
        .. code-block:: python
                from langchain_community.vectorstores import Bagel
                vectorstore = Bagel(cluster_name="langchain_store")
    """
    _LANGCHAIN_DEFAULT_CLUSTER_NAME = "langchain"
[docs]    def __init__(
        self,
        cluster_name: str = _LANGCHAIN_DEFAULT_CLUSTER_NAME,
self,
        cluster_name: str = _LANGCHAIN_DEFAULT_CLUSTER_NAME,
        client_settings: Optional[bagel.config.Settings] = None,
        embedding_function: Optional[Embeddings] = None,
        cluster_metadata: Optional[Dict] = None,
        client: Optional[bagel.Client] = None,
        relevance_score_fn: Optional[Callable[[float], float]] = None,
    ) -> None:
        """Initialize with bagel client"""
        try:
            import bagel
            import bagel.config
        except ImportError:
            raise ImportError("Please install bagel `pip install betabageldb`.")
        if client is not None:
            self._client_settings = client_settings
            self._client = client
        else:
            if client_settings:
                _client_settings = client_settings
            else:
                _client_settings = bagel.config.Settings(
                    bagel_api_impl="rest",
                    bagel_server_host="api.bageldb.ai",
                )
            self._client_settings = _client_settings
            self._client = bagel.Client(_client_settings)
        self._cluster = self._client.get_or_create_cluster(
            name=cluster_name,
            metadata=cluster_metadata,
        )
        self.override_relevance_score_fn = relevance_score_fn
        self._embedding_function = embedding_function
    @property
    def embeddings(self) -> Optional[Embeddings]:
        return self._embedding_function
    @xor_args(("query_texts", "query_embeddings"))
    def __query_cluster(
        self,
        query_texts: Optional[List[str]] = None,
        query_embeddings: Optional[List[List[float]]] = None,
        n_results: int = 4,
        where: Optional[Dict[str, str]] = None,
where: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Query the BagelDB cluster based on the provided parameters."""
        try:
            import bagel  # noqa: F401
        except ImportError:
            raise ImportError("Please install bagel `pip install betabageldb`.")
        return self._cluster.find(
            query_texts=query_texts,
            query_embeddings=query_embeddings,
            n_results=n_results,
            where=where,
            **kwargs,
        )
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        embeddings: Optional[List[List[float]]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """
        Add texts along with their corresponding embeddings and optional
        metadata to the BagelDB cluster.
        Args:
            texts (Iterable[str]): Texts to be added.
            embeddings (Optional[List[float]]): List of embeddingvectors
            metadatas (Optional[List[dict]]): Optional list of metadatas.
            ids (Optional[List[str]]): List of unique ID for the texts.
        Returns:
            List[str]: List of unique ID representing the added texts.
        """
        # creating unique ids if None
        if ids is None:
            ids = [str(uuid.uuid1()) for _ in texts]
        texts = list(texts)
        if self._embedding_function and embeddings is None and texts:
            embeddings = self._embedding_function.embed_documents(texts)
        if metadatas:
            length_diff = len(texts) - len(metadatas)
length_diff = len(texts) - len(metadatas)
            if length_diff:
                metadatas = metadatas + [{}] * length_diff
            empty_ids = []
            non_empty_ids = []
            for idx, metadata in enumerate(metadatas):
                if metadata:
                    non_empty_ids.append(idx)
                else:
                    empty_ids.append(idx)
            if non_empty_ids:
                metadatas = [metadatas[idx] for idx in non_empty_ids]
                texts_with_metadatas = [texts[idx] for idx in non_empty_ids]
                embeddings_with_metadatas = (
                    [embeddings[idx] for idx in non_empty_ids] if embeddings else None
                )
                ids_with_metadata = [ids[idx] for idx in non_empty_ids]
                self._cluster.upsert(
                    embeddings=embeddings_with_metadatas,
                    metadatas=metadatas,
                    documents=texts_with_metadatas,
                    ids=ids_with_metadata,
                )
            if empty_ids:
                texts_without_metadatas = [texts[j] for j in empty_ids]
                embeddings_without_metadatas = (
                    [embeddings[j] for j in empty_ids] if embeddings else None
                )
                ids_without_metadatas = [ids[j] for j in empty_ids]
                self._cluster.upsert(
                    embeddings=embeddings_without_metadatas,
                    documents=texts_without_metadatas,
                    ids=ids_without_metadatas,
                )
        else:
            metadatas = [{}] * len(texts)
            self._cluster.upsert(
                embeddings=embeddings,
                documents=texts,
                metadatas=metadatas,
                ids=ids,
            )
metadatas=metadatas,
                ids=ids,
            )
        return ids
[docs]    def similarity_search(
        self,
        query: str,
        k: int = DEFAULT_K,
        where: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """
        Run a similarity search with BagelDB.
        Args:
            query (str): The query text to search for similar documents/texts.
            k (int): The number of results to return.
            where (Optional[Dict[str, str]]): Metadata filters to narrow down.
        Returns:
            List[Document]: List of documents objects representing
            the documents most similar to the query text.
        """
        docs_and_scores = self.similarity_search_with_score(query, k, where=where)
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = DEFAULT_K,
        where: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """
        Run a similarity search with BagelDB and return documents with their
        corresponding similarity scores.
        Args:
            query (str): The query text to search for similar documents.
            k (int): The number of results to return.
            where (Optional[Dict[str, str]]): Filter using metadata.
        Returns:
            List[Tuple[Document, float]]: List of tuples, each containing a
            Document object representing a similar document and its
            corresponding similarity score.
        """
Document object representing a similar document and its
            corresponding similarity score.
        """
        results = self.__query_cluster(query_texts=[query], n_results=k, where=where)
        return _results_to_docs_and_scores(results)
[docs]    @classmethod
    def from_texts(
        cls: Type[Bagel],
        texts: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        cluster_name: str = _LANGCHAIN_DEFAULT_CLUSTER_NAME,
        client_settings: Optional[bagel.config.Settings] = None,
        cluster_metadata: Optional[Dict] = None,
        client: Optional[bagel.Client] = None,
        text_embeddings: Optional[List[List[float]]] = None,
        **kwargs: Any,
    ) -> Bagel:
        """
        Create and initialize a Bagel instance from list of texts.
        Args:
            texts (List[str]): List of text content to be added.
            cluster_name (str): The name of the BagelDB cluster.
            client_settings (Optional[bagel.config.Settings]): Client settings.
            cluster_metadata (Optional[Dict]): Metadata of the cluster.
            embeddings (Optional[Embeddings]): List of embedding.
            metadatas (Optional[List[dict]]): List of metadata.
            ids (Optional[List[str]]): List of unique ID. Defaults to None.
            client (Optional[bagel.Client]): Bagel client instance.
        Returns:
            Bagel: Bagel vectorstore.
        """
        bagel_cluster = cls(
            cluster_name=cluster_name,
            embedding_function=embedding,
            client_settings=client_settings,
            client=client,
client_settings=client_settings,
            client=client,
            cluster_metadata=cluster_metadata,
            **kwargs,
        )
        _ = bagel_cluster.add_texts(
            texts=texts, embeddings=text_embeddings, metadatas=metadatas, ids=ids
        )
        return bagel_cluster
[docs]    def delete_cluster(self) -> None:
        """Delete the cluster."""
        self._client.delete_cluster(self._cluster.name)
[docs]    def similarity_search_by_vector_with_relevance_scores(
        self,
        query_embeddings: List[float],
        k: int = DEFAULT_K,
        where: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """
        Return docs most similar to embedding vector and similarity score.
        """
        results = self.__query_cluster(
            query_embeddings=query_embeddings, n_results=k, where=where
        )
        return _results_to_docs_and_scores(results)
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = DEFAULT_K,
        where: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs most similar to embedding vector."""
        results = self.__query_cluster(
            query_embeddings=embedding, n_results=k, where=where
        )
        return _results_to_docs(results)
    def _select_relevance_score_fn(self) -> Callable[[float], float]:
        """
        Select and return the appropriate relevance score function based
        on the distance metric used in the BagelDB cluster.
        """
        if self.override_relevance_score_fn:
"""
        if self.override_relevance_score_fn:
            return self.override_relevance_score_fn
        distance = "l2"
        distance_key = "hnsw:space"
        metadata = self._cluster.metadata
        if metadata and distance_key in metadata:
            distance = metadata[distance_key]
        if distance == "cosine":
            return self._cosine_relevance_score_fn
        elif distance == "l2":
            return self._euclidean_relevance_score_fn
        elif distance == "ip":
            return self._max_inner_product_relevance_score_fn
        else:
            raise ValueError(
                "No supported normalization function for distance"
                f" metric of type: {distance}. Consider providing"
                " relevance_score_fn to Bagel constructor."
            )
[docs]    @classmethod
    def from_documents(
        cls: Type[Bagel],
        documents: List[Document],
        embedding: Optional[Embeddings] = None,
        ids: Optional[List[str]] = None,
        cluster_name: str = _LANGCHAIN_DEFAULT_CLUSTER_NAME,
        client_settings: Optional[bagel.config.Settings] = None,
        client: Optional[bagel.Client] = None,
        cluster_metadata: Optional[Dict] = None,
        **kwargs: Any,
    ) -> Bagel:
        """
        Create a Bagel vectorstore from a list of documents.
        Args:
            documents (List[Document]): List of Document objects to add to the
                                        Bagel vectorstore.
            embedding (Optional[List[float]]): List of embedding.
            ids (Optional[List[str]]): List of IDs. Defaults to None.
            cluster_name (str): The name of the BagelDB cluster.
cluster_name (str): The name of the BagelDB cluster.
            client_settings (Optional[bagel.config.Settings]): Client settings.
            client (Optional[bagel.Client]): Bagel client instance.
            cluster_metadata (Optional[Dict]): Metadata associated with the
                                               Bagel cluster. Defaults to None.
        Returns:
            Bagel: Bagel vectorstore.
        """
        texts = [doc.page_content for doc in documents]
        metadatas = [doc.metadata for doc in documents]
        return cls.from_texts(
            texts=texts,
            embedding=embedding,
            metadatas=metadatas,
            ids=ids,
            cluster_name=cluster_name,
            client_settings=client_settings,
            client=client,
            cluster_metadata=cluster_metadata,
            **kwargs,
        )
[docs]    def update_document(self, document_id: str, document: Document) -> None:
        """Update a document in the cluster.
        Args:
            document_id (str): ID of the document to update.
            document (Document): Document to update.
        """
        text = document.page_content
        metadata = document.metadata
        self._cluster.update(
            ids=[document_id],
            documents=[text],
            metadatas=[metadata],
        )
[docs]    def get(
        self,
        ids: Optional[OneOrMany[ID]] = None,
        where: Optional[Where] = None,
        limit: Optional[int] = None,
        offset: Optional[int] = None,
        where_document: Optional[WhereDocument] = None,
        include: Optional[List[str]] = None,
    ) -> Dict[str, Any]:
        """Gets the collection."""
        kwargs = {
"""Gets the collection."""
        kwargs = {
            "ids": ids,
            "where": where,
            "limit": limit,
            "offset": offset,
            "where_document": where_document,
        }
        if include is not None:
            kwargs["include"] = include
        return self._cluster.get(**kwargs)
[docs]    def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> None:
        """
        Delete by IDs.
        Args:
            ids: List of ids to delete.
        """
        self._cluster.delete(ids=ids)