viruPVS/langchain-docs huggingface dataset:
"""
        if self.awadb_client is None:
            raise ValueError("AwaDB client is None!!!")
        results: List[Document] = []
        if embedding is None:
            return results
        not_include_fields: set = {"_id", "score"}
        retrieved_docs = self.similarity_search_by_vector(
            embedding,
            fetch_k,
            text_in_page_content=text_in_page_content,
            meta_filter=meta_filter,
            not_include_fields_in_metadata=not_include_fields,
        )
        top_embeddings = []
        for doc in retrieved_docs:
            top_embeddings.append(doc.metadata["text_embedding"])
        selected_docs = maximal_marginal_relevance(
            np.array(embedding, dtype=np.float32), embedding_list=top_embeddings
        )
        for s_id in selected_docs:
            if "text_embedding" in retrieved_docs[s_id].metadata:
                del retrieved_docs[s_id].metadata["text_embedding"]
                results.append(retrieved_docs[s_id])
        return results
[docs]    def get(
        self,
        ids: Optional[List[str]] = None,
        text_in_page_content: Optional[str] = None,
        meta_filter: Optional[dict] = None,
        not_include_fields: Optional[Set[str]] = None,
        limit: Optional[int] = None,
        **kwargs: Any,
    ) -> Dict[str, Document]:
        """Return docs according ids.
        Args:
            ids: The ids of the embedding vectors.
            text_in_page_content: Filter by the text in page_content of Document.
            meta_filter: Filter by any metadata of the document.
            not_include_fields: Not pack the specified fields of each document.
            limit: The number of documents to return. Defaults to 5. Optional.
limit: The number of documents to return. Defaults to 5. Optional.
        Returns:
            Documents which satisfy the input conditions.
        """
        if self.awadb_client is None:
            raise ValueError("AwaDB client is None!!!")
        docs_detail = self.awadb_client.Get(
            ids=ids,
            text_in_page_content=text_in_page_content,
            meta_filter=meta_filter,
            not_include_fields=not_include_fields,
            limit=limit,
        )
        results: Dict[str, Document] = {}
        for doc_detail in docs_detail:
            content = ""
            meta_info = {}
            for field in doc_detail:
                if field == "embedding_text":
                    content = doc_detail[field]
                    continue
                elif field == "text_embedding" or field == "_id":
                    continue
                meta_info[field] = doc_detail[field]
            doc = Document(page_content=content, metadata=meta_info)
            results[doc_detail["_id"]] = doc
        return results
[docs]    def delete(
        self,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> Optional[bool]:
        """Delete the documents which have the specified ids.
        Args:
            ids: The ids of the embedding vectors.
            **kwargs: Other keyword arguments that subclasses might use.
        Returns:
            Optional[bool]: True if deletion is successful.
            False otherwise, None if not implemented.
        """
        if self.awadb_client is None:
            raise ValueError("AwaDB client is None!!!")
        ret: Optional[bool] = None
        if ids is None or ids.__len__() == 0:
            return ret
        ret = self.awadb_client.Delete(ids)
return ret
        ret = self.awadb_client.Delete(ids)
        return ret
[docs]    def update(
        self,
        ids: List[str],
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Update the documents which have the specified ids.
        Args:
            ids: The id list of the updating embedding vector.
            texts: The texts of the updating documents.
            metadatas: The metadatas of the updating documents.
        Returns:
            the ids of the updated documents.
        """
        if self.awadb_client is None:
            raise ValueError("AwaDB client is None!!!")
        return self.awadb_client.UpdateTexts(
            ids=ids, text_field_name="embedding_text", texts=texts, metadatas=metadatas
        )
[docs]    def create_table(
        self,
        table_name: str,
        **kwargs: Any,
    ) -> bool:
        """Create a new table."""
        if self.awadb_client is None:
            return False
        ret = self.awadb_client.Create(table_name)
        if ret:
            self.using_table_name = table_name
        return ret
[docs]    def use(
        self,
        table_name: str,
        **kwargs: Any,
    ) -> bool:
        """Use the specified table. Don't know the tables, please invoke list_tables."""
        if self.awadb_client is None:
            return False
        ret = self.awadb_client.Use(table_name)
        if ret:
            self.using_table_name = table_name
        return ret
[docs]    def list_tables(
        self,
return ret
[docs]    def list_tables(
        self,
        **kwargs: Any,
    ) -> List[str]:
        """List all the tables created by the client."""
        if self.awadb_client is None:
            return []
        return self.awadb_client.ListAllTables()
[docs]    def get_current_table(
        self,
        **kwargs: Any,
    ) -> str:
        """Get the current table."""
        return self.using_table_name
[docs]    @classmethod
    def from_texts(
        cls: Type[AwaDB],
        texts: List[str],
        embedding: Optional[Embeddings] = None,
        metadatas: Optional[List[dict]] = None,
        table_name: str = _DEFAULT_TABLE_NAME,
        log_and_data_dir: Optional[str] = None,
        client: Optional[awadb.Client] = None,
        **kwargs: Any,
    ) -> AwaDB:
        """Create an AwaDB vectorstore from a raw documents.
        Args:
            texts (List[str]): List of texts to add to the table.
            embedding (Optional[Embeddings]): Embedding function. Defaults to None.
            metadatas (Optional[List[dict]]): List of metadatas. Defaults to None.
            table_name (str): Name of the table to create.
            log_and_data_dir (Optional[str]): Directory of logging and persistence.
            client (Optional[awadb.Client]): AwaDB client
        Returns:
            AwaDB: AwaDB vectorstore.
        """
        awadb_client = cls(
            table_name=table_name,
            embedding=embedding,
            log_and_data_dir=log_and_data_dir,
            client=client,
        )
log_and_data_dir=log_and_data_dir,
            client=client,
        )
        awadb_client.add_texts(texts=texts, metadatas=metadatas)
        return awadb_client
[docs]    @classmethod
    def from_documents(
        cls: Type[AwaDB],
        documents: List[Document],
        embedding: Optional[Embeddings] = None,
        table_name: str = _DEFAULT_TABLE_NAME,
        log_and_data_dir: Optional[str] = None,
        client: Optional[awadb.Client] = None,
        **kwargs: Any,
    ) -> AwaDB:
        """Create an AwaDB vectorstore from a list of documents.
        If a log_and_data_dir specified, the table will be persisted there.
        Args:
            documents (List[Document]): List of documents to add to the vectorstore.
            embedding (Optional[Embeddings]): Embedding function. Defaults to None.
            table_name (str): Name of the table to create.
            log_and_data_dir (Optional[str]): Directory to persist the table.
            client (Optional[awadb.Client]): AwaDB client.
            Any: Any possible parameters in the future
        Returns:
            AwaDB: AwaDB vectorstore.
        """
        texts = [doc.page_content for doc in documents]
        metadatas = [doc.metadata for doc in documents]
        return cls.from_texts(
            texts=texts,
            embedding=embedding,
            metadatas=metadatas,
            table_name=table_name,
            log_and_data_dir=log_and_data_dir,
            client=client,
        )
Source code for langchain_community.vectorstores.dashvector
from __future__ import annotations
import logging
import uuid
from typing import (
    Any,
    Iterable,
    List,
    Optional,
    Tuple,
)
import numpy as np
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.utils import get_from_env
from langchain_core.vectorstores import VectorStore
from langchain_community.vectorstores.utils import maximal_marginal_relevance
logger = logging.getLogger(__name__)
[docs]class DashVector(VectorStore):
    """`DashVector` vector store.
    To use, you should have the ``dashvector`` python package installed.
    Example:
        .. code-block:: python
            from langchain_community.vectorstores import DashVector
            from langchain_community.embeddings.openai import OpenAIEmbeddings
            import dashvector
            client = dashvector.Client(api_key="***")
            client.create("langchain", dimension=1024)
            collection = client.get("langchain")
            embeddings = OpenAIEmbeddings()
            vectorstore = DashVector(collection, embeddings.embed_query, "text")
    """
[docs]    def __init__(
        self,
        collection: Any,
        embedding: Embeddings,
        text_field: str,
    ):
        """Initialize with DashVector collection."""
        try:
            import dashvector
        except ImportError:
            raise ValueError(
                "Could not import dashvector python package. "
                "Please install it with `pip install dashvector`."
            )
        if not isinstance(collection, dashvector.Collection):
            raise ValueError(
                f"collection should be an instance of dashvector.Collection, "
                f"bug got {type(collection)}"
            )
f"bug got {type(collection)}"
            )
        self._collection = collection
        self._embedding = embedding
        self._text_field = text_field
    def _similarity_search_with_score_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[str] = None,
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query vector, along with scores"""
        # query by vector
        ret = self._collection.query(embedding, topk=k, filter=filter)
        if not ret:
            raise ValueError(
                f"Fail to query docs by vector, error {self._collection.message}"
            )
        docs = []
        for doc in ret:
            metadata = doc.fields
            text = metadata.pop(self._text_field)
            score = doc.score
            docs.append((Document(page_content=text, metadata=metadata), score))
        return docs
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        batch_size: int = 25,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of ids associated with the texts.
            batch_size: Optional batch size to upsert docs.
            kwargs: vectorstore specific parameters
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
List of ids from adding the texts into the vectorstore.
        """
        ids = ids or [str(uuid.uuid4().hex) for _ in texts]
        text_list = list(texts)
        for i in range(0, len(text_list), batch_size):
            # batch end
            end = min(i + batch_size, len(text_list))
            batch_texts = text_list[i:end]
            batch_ids = ids[i:end]
            batch_embeddings = self._embedding.embed_documents(list(batch_texts))
            # batch metadatas
            if metadatas:
                batch_metadatas = metadatas[i:end]
            else:
                batch_metadatas = [{} for _ in range(i, end)]
            for metadata, text in zip(batch_metadatas, batch_texts):
                metadata[self._text_field] = text
            # batch upsert to collection
            docs = list(zip(batch_ids, batch_embeddings, batch_metadatas))
            ret = self._collection.upsert(docs)
            if not ret:
                raise ValueError(
                    f"Fail to upsert docs to dashvector vector database,"
                    f"Error: {ret.message}"
                )
        return ids
[docs]    def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> bool:
        """Delete by vector ID.
        Args:
            ids: List of ids to delete.
        Returns:
            True if deletion is successful,
            False otherwise.
        """
        return bool(self._collection.delete(ids))
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
        filter: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
**kwargs: Any,
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to search documents similar to.
            k: Number of documents to return. Default to 4.
            filter: Doc fields filter conditions that meet the SQL where clause
                    specification.
        Returns:
            List of Documents most similar to the query text.
        """
        docs_and_scores = self.similarity_search_with_relevance_scores(query, k, filter)
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        filter: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query text , alone with relevance scores.
        Less is more similar, more is more dissimilar.
        Args:
            query: input text
            k: Number of Documents to return. Defaults to 4.
            filter: Doc fields filter conditions that meet the SQL where clause
                    specification.
        Returns:
            List of Tuples of (doc, similarity_score)
        """
        embedding = self._embedding.embed_query(query)
        return self._similarity_search_with_score_by_vector(
            embedding, k=k, filter=filter
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        filter: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs most similar to embedding vector.
        Args:
"""Return docs most similar to embedding vector.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            filter: Doc fields filter conditions that meet the SQL where clause
                    specification.
        Returns:
            List of Documents most similar to the query vector.
        """
        docs_and_scores = self._similarity_search_with_score_by_vector(
            embedding, k, filter
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
            filter: Doc fields filter conditions that meet the SQL where clause
                    specification.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding = self._embedding.embed_query(query)
        return self.max_marginal_relevance_search_by_vector(
return self.max_marginal_relevance_search_by_vector(
            embedding, k, fetch_k, lambda_mult, filter
        )
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
            filter: Doc fields filter conditions that meet the SQL where clause
                    specification.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        # query by vector
        ret = self._collection.query(
            embedding, topk=fetch_k, filter=filter, include_vector=True
        )
        if not ret:
            raise ValueError(
                f"Fail to query docs by vector, error {self._collection.message}"
            )
        candidate_embeddings = [doc.vector for doc in ret]
        mmr_selected = maximal_marginal_relevance(
            np.array(embedding), candidate_embeddings, lambda_mult, k
        )
np.array(embedding), candidate_embeddings, lambda_mult, k
        )
        metadatas = [ret.output[i].fields for i in mmr_selected]
        return [
            Document(page_content=metadata.pop(self._text_field), metadata=metadata)
            for metadata in metadatas
        ]
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        dashvector_api_key: Optional[str] = None,
        dashvector_endpoint: Optional[str] = None,
        collection_name: str = "langchain",
        text_field: str = "text",
        batch_size: int = 25,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> DashVector:
        """Return DashVector VectorStore initialized from texts and embeddings.
        This is the quick way to get started with dashvector vector store.
        Example:
            .. code-block:: python
            from langchain_community.vectorstores import DashVector
            from langchain_community.embeddings import OpenAIEmbeddings
            import dashvector
            embeddings = OpenAIEmbeddings()
            dashvector = DashVector.from_documents(
                docs,
                embeddings,
                dashvector_api_key="{DASHVECTOR_API_KEY}"
            )
        """
        try:
            import dashvector
        except ImportError:
            raise ValueError(
                "Could not import dashvector python package. "
                "Please install it with `pip install dashvector`."
            )
        dashvector_api_key = dashvector_api_key or get_from_env(
            "dashvector_api_key", "DASHVECTOR_API_KEY"
        )
"dashvector_api_key", "DASHVECTOR_API_KEY"
        )
        dashvector_endpoint = dashvector_endpoint or get_from_env(
            "dashvector_endpoint",
            "DASHVECTOR_ENDPOINT",
            default="dashvector.cn-hangzhou.aliyuncs.com",
        )
        dashvector_client = dashvector.Client(
            api_key=dashvector_api_key, endpoint=dashvector_endpoint
        )
        dashvector_client.delete(collection_name)
        collection = dashvector_client.get(collection_name)
        if not collection:
            dim = len(embedding.embed_query(texts[0]))
            # create collection if not existed
            resp = dashvector_client.create(collection_name, dimension=dim)
            if resp:
                collection = dashvector_client.get(collection_name)
            else:
                raise ValueError(
                    "Fail to create collection. " f"Error: {resp.message}."
                )
        dashvector_vector_db = cls(collection, embedding, text_field)
        dashvector_vector_db.add_texts(texts, metadatas, ids, batch_size)
        return dashvector_vector_db
Source code for langchain_community.vectorstores.supabase
from __future__ import annotations
import uuid
from itertools import repeat
from typing import (
    TYPE_CHECKING,
    Any,
    Dict,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
    Union,
)
import numpy as np
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore
from langchain_community.vectorstores.utils import maximal_marginal_relevance
if TYPE_CHECKING:
    import supabase
[docs]class SupabaseVectorStore(VectorStore):
    """`Supabase Postgres` vector store.
    It assumes you have the `pgvector`
    extension installed and a `match_documents` (or similar) function. For more details:
    https://integrations.langchain.com/vectorstores?integration_name=SupabaseVectorStore
    You can implement your own `match_documents` function in order to limit the search
    space to a subset of documents based on your own authorization or business logic.
    Note that the Supabase Python client does not yet support async operations.
    If you'd like to use `max_marginal_relevance_search`, please review the instructions
    below on modifying the `match_documents` function to return matched embeddings.
    Examples:
    .. code-block:: python
        from langchain_community.embeddings.openai import OpenAIEmbeddings
        from langchain_core.documents import Document
        from langchain_community.vectorstores import SupabaseVectorStore
        from supabase.client import create_client
        docs = [
            Document(page_content="foo", metadata={"id": 1}),
        ]
        embeddings = OpenAIEmbeddings()