viruPVS/langchain-docs huggingface dataset:
"""Empty the collection of all its stored entries."""
        self._drop_collection()
        self._provision_collection()
        return None
[docs]    def delete_by_document_id(self, document_id: str) -> bool:
        """
        Remove a single document from the store, given its document_id (str).
        Return True if a document has indeed been deleted, False if ID not found.
        """
        deletion_response = self.collection.delete(document_id)
        return ((deletion_response or {}).get("status") or {}).get(
            "deletedCount", 0
        ) == 1
[docs]    def delete(
        self,
        ids: Optional[List[str]] = None,
        concurrency: Optional[int] = None,
        **kwargs: Any,
    ) -> Optional[bool]:
        """Delete by vector ids.
        Args:
            ids (Optional[List[str]]): List of ids to delete.
            concurrency (Optional[int]): max number of threads issuing
                single-doc delete requests. Defaults to instance-level setting.
        Returns:
            Optional[bool]: True if deletion is successful,
                False otherwise, None if not implemented.
        """
        if kwargs:
            warnings.warn(
                "Method 'delete' of AstraDB vector store invoked with "
                f"unsupported arguments ({', '.join(sorted(kwargs.keys()))}), "
                "which will be ignored."
            )
        if ids is None:
            raise ValueError("No ids provided to delete.")
        _max_workers = concurrency or self.bulk_delete_concurrency
        with ThreadPoolExecutor(max_workers=_max_workers) as tpe:
            _ = list(
                tpe.map(
                    self.delete_by_document_id,
                    ids,
                )
            )
        return True
ids,
                )
            )
        return True
[docs]    def delete_collection(self) -> None:
        """
        Completely delete the collection from the database (as opposed
        to 'clear()', which empties it only).
        Stored data is lost and unrecoverable, resources are freed.
        Use with caution.
        """
        self._drop_collection()
        return None
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        *,
        batch_size: Optional[int] = None,
        batch_concurrency: Optional[int] = None,
        overwrite_concurrency: Optional[int] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run texts through the embeddings and add them to the vectorstore.
        If passing explicit ids, those entries whose id is in the store already
        will be replaced.
        Args:
            texts (Iterable[str]): Texts to add to the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadatas.
            ids (Optional[List[str]], optional): Optional list of ids.
            batch_size (Optional[int]): Number of documents in each API call.
                Check the underlying Astra DB HTTP API specs for the max value
                (20 at the time of writing this). If not provided, defaults
                to the instance-level setting.
            batch_concurrency (Optional[int]): number of threads to process
                insertion batches concurrently. Defaults to instance-level
                setting if not provided.
            overwrite_concurrency (Optional[int]):  number of threads to process
                pre-existing documents in each batch (which require individual
pre-existing documents in each batch (which require individual
                API calls). Defaults to instance-level setting if not provided.
        A note on metadata: there are constraints on the allowed field names
        in this dictionary, coming from the underlying Astra DB API.
        For instance, the `$` (dollar sign) cannot be used in the dict keys.
        See this document for details:
            docs.datastax.com/en/astra-serverless/docs/develop/dev-with-json.html
        Returns:
            List[str]: List of ids of the added texts.
        """
        if kwargs:
            warnings.warn(
                "Method 'add_texts' of AstraDB vector store invoked with "
                f"unsupported arguments ({', '.join(sorted(kwargs.keys()))}), "
                "which will be ignored."
            )
        _texts = list(texts)
        if ids is None:
            ids = [uuid.uuid4().hex for _ in _texts]
        if metadatas is None:
            metadatas = [{} for _ in _texts]
        #
        embedding_vectors = self.embedding.embed_documents(_texts)
        documents_to_insert = [
            {
                "content": b_txt,
                "_id": b_id,
                "$vector": b_emb,
                "metadata": b_md,
            }
            for b_txt, b_emb, b_id, b_md in zip(
                _texts,
                embedding_vectors,
                ids,
                metadatas,
            )
        ]
        # make unique by id, keeping the last
        uniqued_documents_to_insert = _unique_list(
            documents_to_insert[::-1],
            lambda document: document["_id"],
        )[::-1]
        all_ids = []
)[::-1]
        all_ids = []
        def _handle_batch(document_batch: List[DocDict]) -> List[str]:
            im_result = self.collection.insert_many(
                documents=document_batch,
                options={"ordered": False},
                partial_failures_allowed=True,
            )
            if "status" not in im_result:
                raise ValueError(
                    f"API Exception while running bulk insertion: {str(im_result)}"
                )
            batch_inserted = im_result["status"]["insertedIds"]
            # estimation of the preexisting documents that failed
            missed_inserted_ids = {
                document["_id"] for document in document_batch
            } - set(batch_inserted)
            errors = im_result.get("errors", [])
            # careful for other sources of error other than "doc already exists"
            num_errors = len(errors)
            unexpected_errors = any(
                error.get("errorCode") != "DOCUMENT_ALREADY_EXISTS" for error in errors
            )
            if num_errors != len(missed_inserted_ids) or unexpected_errors:
                raise ValueError(
                    f"API Exception while running bulk insertion: {str(errors)}"
                )
            # deal with the missing insertions as upserts
            missing_from_batch = [
                document
                for document in document_batch
                if document["_id"] in missed_inserted_ids
            ]
            def _handle_missing_document(missing_document: DocDict) -> str:
                replacement_result = self.collection.find_one_and_replace(
                    filter={"_id": missing_document["_id"]},
                    replacement=missing_document,
                )
                return replacement_result["data"]["document"]["_id"]
            _u_max_workers = (
                overwrite_concurrency or self.bulk_insert_overwrite_concurrency
            )
overwrite_concurrency or self.bulk_insert_overwrite_concurrency
            )
            with ThreadPoolExecutor(max_workers=_u_max_workers) as tpe2:
                batch_replaced = list(
                    tpe2.map(
                        _handle_missing_document,
                        missing_from_batch,
                    )
                )
            upsert_ids = batch_inserted + batch_replaced
            return upsert_ids
        _b_max_workers = batch_concurrency or self.bulk_insert_batch_concurrency
        with ThreadPoolExecutor(max_workers=_b_max_workers) as tpe:
            all_ids_nested = tpe.map(
                _handle_batch,
                batch_iterate(
                    batch_size or self.batch_size,
                    uniqued_documents_to_insert,
                ),
            )
        all_ids = [iid for id_list in all_ids_nested for iid in id_list]
        return all_ids
[docs]    def similarity_search_with_score_id_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
    ) -> List[Tuple[Document, float, str]]:
        """Return docs most similar to embedding vector.
        Args:
            embedding (str): Embedding to look up documents similar to.
            k (int): Number of Documents to return. Defaults to 4.
        Returns:
            List of (Document, score, id), the most similar to the query vector.
        """
        metadata_parameter = self._filter_to_metadata(filter)
        #
        hits = list(
            self.collection.paginated_find(
                filter=metadata_parameter,
                sort={"$vector": embedding},
                options={"limit": k, "includeSimilarity": True},
                projection={
                    "_id": 1,
projection={
                    "_id": 1,
                    "content": 1,
                    "metadata": 1,
                },
            )
        )
        #
        return [
            (
                Document(
                    page_content=hit["content"],
                    metadata=hit["metadata"],
                ),
                hit["$similarity"],
                hit["_id"],
            )
            for hit in hits
        ]
[docs]    def similarity_search_with_score_id(
        self,
        query: str,
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
    ) -> List[Tuple[Document, float, str]]:
        embedding_vector = self.embedding.embed_query(query)
        return self.similarity_search_with_score_id_by_vector(
            embedding=embedding_vector,
            k=k,
            filter=filter,
        )
[docs]    def similarity_search_with_score_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to embedding vector.
        Args:
            embedding (str): Embedding to look up documents similar to.
            k (int): Number of Documents to return. Defaults to 4.
        Returns:
            List of (Document, score), the most similar to the query vector.
        """
        return [
            (doc, score)
            for (doc, score, doc_id) in self.similarity_search_with_score_id_by_vector(
                embedding=embedding,
                k=k,
                filter=filter,
            )
        ]
[docs]    def similarity_search(
)
        ]
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        embedding_vector = self.embedding.embed_query(query)
        return self.similarity_search_by_vector(
            embedding_vector,
            k,
            filter=filter,
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        return [
            doc
            for doc, _ in self.similarity_search_with_score_by_vector(
                embedding,
                k,
                filter=filter,
            )
        ]
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 4,
        filter: Optional[Dict[str, str]] = None,
    ) -> List[Tuple[Document, float]]:
        embedding_vector = self.embedding.embed_query(query)
        return self.similarity_search_with_score_by_vector(
            embedding_vector,
            k,
            filter=filter,
        )
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
**kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        metadata_parameter = self._filter_to_metadata(filter)
        prefetch_hits = list(
            self.collection.paginated_find(
                filter=metadata_parameter,
                sort={"$vector": embedding},
                options={"limit": fetch_k, "includeSimilarity": True},
                projection={
                    "_id": 1,
                    "content": 1,
                    "metadata": 1,
                    "$vector": 1,
                },
            )
        )
        mmr_chosen_indices = maximal_marginal_relevance(
            np.array(embedding, dtype=np.float32),
            [prefetch_hit["$vector"] for prefetch_hit in prefetch_hits],
            k=k,
            lambda_mult=lambda_mult,
        )
        mmr_hits = [
            prefetch_hit
            for prefetch_index, prefetch_hit in enumerate(prefetch_hits)
            if prefetch_index in mmr_chosen_indices
        ]
        return [
            Document(
                page_content=hit["content"],
                metadata=hit["metadata"],
            )
            for hit in mmr_hits
        ]
)
            for hit in mmr_hits
        ]
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[Dict[str, str]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query (str): Text to look up documents similar to.
            k (int = 4): Number of Documents to return.
            fetch_k (int = 20): Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult (float = 0.5): Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Optional.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding_vector = self.embedding.embed_query(query)
        return self.max_marginal_relevance_search_by_vector(
            embedding_vector,
            k,
            fetch_k,
            lambda_mult=lambda_mult,
            filter=filter,
        )
[docs]    @classmethod
    def from_texts(
        cls: Type[ADBVST],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> ADBVST:
        """Create an Astra DB vectorstore from raw texts.
"""Create an Astra DB vectorstore from raw texts.
        Args:
            texts (List[str]): the texts to insert.
            embedding (Embeddings): the embedding function to use in the store.
            metadatas (Optional[List[dict]]): metadata dicts for the texts.
            ids (Optional[List[str]]): ids to associate to the texts.
            *Additional arguments*: you can pass any argument that you would
                to 'add_texts' and/or to the 'AstraDB' class constructor
                (see these methods for details). These arguments will be
                routed to the respective methods as they are.
        Returns:
            an `AstraDb` vectorstore.
        """
        known_kwargs = {
            "collection_name",
            "token",
            "api_endpoint",
            "astra_db_client",
            "namespace",
            "metric",
            "batch_size",
            "bulk_insert_batch_concurrency",
            "bulk_insert_overwrite_concurrency",
            "bulk_delete_concurrency",
            "batch_concurrency",
            "overwrite_concurrency",
        }
        if kwargs:
            unknown_kwargs = set(kwargs.keys()) - known_kwargs
            if unknown_kwargs:
                warnings.warn(
                    "Method 'from_texts' of AstraDB vector store invoked with "
                    f"unsupported arguments ({', '.join(sorted(unknown_kwargs))}), "
                    "which will be ignored."
                )
        collection_name: str = kwargs["collection_name"]
        token = kwargs.get("token")
        api_endpoint = kwargs.get("api_endpoint")
        astra_db_client = kwargs.get("astra_db_client")
        namespace = kwargs.get("namespace")
        metric = kwargs.get("metric")
        astra_db_store = cls(
            embedding=embedding,
astra_db_store = cls(
            embedding=embedding,
            collection_name=collection_name,
            token=token,
            api_endpoint=api_endpoint,
            astra_db_client=astra_db_client,
            namespace=namespace,
            metric=metric,
            batch_size=kwargs.get("batch_size"),
            bulk_insert_batch_concurrency=kwargs.get("bulk_insert_batch_concurrency"),
            bulk_insert_overwrite_concurrency=kwargs.get(
                "bulk_insert_overwrite_concurrency"
            ),
            bulk_delete_concurrency=kwargs.get("bulk_delete_concurrency"),
        )
        astra_db_store.add_texts(
            texts=texts,
            metadatas=metadatas,
            ids=ids,
            batch_size=kwargs.get("batch_size"),
            batch_concurrency=kwargs.get("batch_concurrency"),
            overwrite_concurrency=kwargs.get("overwrite_concurrency"),
        )
        return astra_db_store
[docs]    @classmethod
    def from_documents(
        cls: Type[ADBVST],
        documents: List[Document],
        embedding: Embeddings,
        **kwargs: Any,
    ) -> ADBVST:
        """Create an Astra DB vectorstore from a document list.
        Utility method that defers to 'from_texts' (see that one).
        Args: see 'from_texts', except here you have to supply 'documents'
            in place of 'texts' and 'metadatas'.
        Returns:
            an `AstraDB` vectorstore.
        """
        return super().from_documents(documents, embedding, **kwargs)
Source code for langchain_community.vectorstores.annoy
from __future__ import annotations
import os
import pickle
import uuid
from configparser import ConfigParser
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
import numpy as np
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore
from langchain_community.docstore.base import Docstore
from langchain_community.docstore.in_memory import InMemoryDocstore
from langchain_community.vectorstores.utils import maximal_marginal_relevance
INDEX_METRICS = frozenset(["angular", "euclidean", "manhattan", "hamming", "dot"])
DEFAULT_METRIC = "angular"
[docs]def dependable_annoy_import() -> Any:
    """Import annoy if available, otherwise raise error."""
    try:
        import annoy
    except ImportError:
        raise ImportError(
            "Could not import annoy python package. "
            "Please install it with `pip install --user annoy` "
        )
    return annoy
[docs]class Annoy(VectorStore):
    """`Annoy` vector store.
    To use, you should have the ``annoy`` python package installed.
    Example:
        .. code-block:: python
            from langchain_community.vectorstores import Annoy
            db = Annoy(embedding_function, index, docstore, index_to_docstore_id)
    """
[docs]    def __init__(
        self,
        embedding_function: Callable,
        index: Any,
        metric: str,
        docstore: Docstore,
        index_to_docstore_id: Dict[int, str],
    ):
        """Initialize with necessary components."""
        self.embedding_function = embedding_function
):
        """Initialize with necessary components."""
        self.embedding_function = embedding_function
        self.index = index
        self.metric = metric
        self.docstore = docstore
        self.index_to_docstore_id = index_to_docstore_id
    @property
    def embeddings(self) -> Optional[Embeddings]:
        # TODO: Accept embedding object directly
        return None
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        raise NotImplementedError(
            "Annoy does not allow to add new data once the index is build."
        )
[docs]    def process_index_results(
        self, idxs: List[int], dists: List[float]
    ) -> List[Tuple[Document, float]]:
        """Turns annoy results into a list of documents and scores.
        Args:
            idxs: List of indices of the documents in the index.
            dists: List of distances of the documents in the index.
        Returns:
            List of Documents and scores.
        """
        docs = []
        for idx, dist in zip(idxs, dists):
            _id = self.index_to_docstore_id[idx]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append((doc, dist))
        return docs
[docs]    def similarity_search_with_score_by_vector(
        self, embedding: List[float], k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        idxs, dists = self.index.get_nns_by_vector(
            embedding, k, search_k=search_k, include_distances=True
        )
        return self.process_index_results(idxs, dists)
[docs]    def similarity_search_with_score_by_index(
        self, docstore_index: int, k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        idxs, dists = self.index.get_nns_by_item(
            docstore_index, k, search_k=search_k, include_distances=True
        )
        return self.process_index_results(idxs, dists)
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4, search_k: int = -1
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.