viruPVS/langchain-docs huggingface dataset:
Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query and score for each
        """
        embedding = self.embedding_function(query)
        docs = self.similarity_search_with_score_by_vector(embedding, k, search_k)
        return docs
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to embedding vector.
        Args:
            embedding: Embedding to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_vector(
            embedding, k, search_k
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search_by_index(
        self, docstore_index: int, k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to docstore_index.
        Args:
            docstore_index: Index of document in docstore
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
to n_trees * n if not provided
        Returns:
            List of Documents most similar to the embedding.
        """
        docs_and_scores = self.similarity_search_with_score_by_index(
            docstore_index, k, search_k
        )
        return [doc for doc, _ in docs_and_scores]
[docs]    def similarity_search(
        self, query: str, k: int = 4, search_k: int = -1, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            search_k: inspect up to search_k nodes which defaults
                to n_trees * n if not provided
        Returns:
            List of Documents most similar to the query.
        """
        docs_and_scores = self.similarity_search_with_score(query, k, search_k)
        return [doc for doc, _ in docs_and_scores]
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            embedding: Embedding to look up documents similar to.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            k: Number of Documents to return. Defaults to 4.
            lambda_mult: Number between 0 and 1 that determines the degree
lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        idxs = self.index.get_nns_by_vector(
            embedding, fetch_k, search_k=-1, include_distances=False
        )
        embeddings = [self.index.get_item_vector(i) for i in idxs]
        mmr_selected = maximal_marginal_relevance(
            np.array([embedding], dtype=np.float32),
            embeddings,
            k=k,
            lambda_mult=lambda_mult,
        )
        # ignore the -1's if not enough docs are returned/indexed
        selected_indices = [idxs[i] for i in mmr_selected if i != -1]
        docs = []
        for i in selected_indices:
            _id = self.index_to_docstore_id[i]
            doc = self.docstore.search(_id)
            if not isinstance(doc, Document):
                raise ValueError(f"Could not find document for id {_id}, got {doc}")
            docs.append(doc)
        return docs
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        """Return docs selected using the maximal marginal relevance.
        Maximal marginal relevance optimizes for similarity to query AND diversity
        among selected documents.
        Args:
            query: Text to look up documents similar to.
Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
            fetch_k: Number of Documents to fetch to pass to MMR algorithm.
            lambda_mult: Number between 0 and 1 that determines the degree
                        of diversity among the results with 0 corresponding
                        to maximum diversity and 1 to minimum diversity.
                        Defaults to 0.5.
        Returns:
            List of Documents selected by maximal marginal relevance.
        """
        embedding = self.embedding_function(query)
        docs = self.max_marginal_relevance_search_by_vector(
            embedding, k, fetch_k, lambda_mult=lambda_mult
        )
        return docs
    @classmethod
    def __from(
        cls,
        texts: List[str],
        embeddings: List[List[float]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        if metric not in INDEX_METRICS:
            raise ValueError(
                (
                    f"Unsupported distance metric: {metric}. "
                    f"Expected one of {list(INDEX_METRICS)}"
                )
            )
        annoy = dependable_annoy_import()
        if not embeddings:
            raise ValueError("embeddings must be provided to build AnnoyIndex")
        f = len(embeddings[0])
        index = annoy.AnnoyIndex(f, metric=metric)
        for i, emb in enumerate(embeddings):
            index.add_item(i, emb)
        index.build(trees, n_jobs=n_jobs)
        documents = []
index.build(trees, n_jobs=n_jobs)
        documents = []
        for i, text in enumerate(texts):
            metadata = metadatas[i] if metadatas else {}
            documents.append(Document(page_content=text, metadata=metadata))
        index_to_id = {i: str(uuid.uuid4()) for i in range(len(documents))}
        docstore = InMemoryDocstore(
            {index_to_id[i]: doc for i, doc in enumerate(documents)}
        )
        return cls(embedding.embed_query, index, metric, docstore, index_to_id)
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        """Construct Annoy wrapper from raw documents.
        Args:
            texts: List of documents to index.
            embedding: Embedding function to use.
            metadatas: List of metadata dictionaries to associate with documents.
            metric: Metric to use for indexing. Defaults to "angular".
            trees: Number of trees to use for indexing. Defaults to 100.
            n_jobs: Number of jobs to use for indexing. Defaults to -1.
        This is a user friendly interface that:
            1. Embeds documents.
            2. Creates an in memory docstore
            3. Initializes the Annoy database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain_community.vectorstores import Annoy
.. code-block:: python
                from langchain_community.vectorstores import Annoy
                from langchain_community.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                index = Annoy.from_texts(texts, embeddings)
        """
        embeddings = embedding.embed_documents(texts)
        return cls.__from(
            texts, embeddings, embedding, metadatas, metric, trees, n_jobs, **kwargs
        )
[docs]    @classmethod
    def from_embeddings(
        cls,
        text_embeddings: List[Tuple[str, List[float]]],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        metric: str = DEFAULT_METRIC,
        trees: int = 100,
        n_jobs: int = -1,
        **kwargs: Any,
    ) -> Annoy:
        """Construct Annoy wrapper from embeddings.
        Args:
            text_embeddings: List of tuples of (text, embedding)
            embedding: Embedding function to use.
            metadatas: List of metadata dictionaries to associate with documents.
            metric: Metric to use for indexing. Defaults to "angular".
            trees: Number of trees to use for indexing. Defaults to 100.
            n_jobs: Number of jobs to use for indexing. Defaults to -1
        This is a user friendly interface that:
            1. Creates an in memory docstore with provided embeddings
            2. Initializes the Annoy database
        This is intended to be a quick way to get started.
        Example:
            .. code-block:: python
                from langchain_community.vectorstores import Annoy
                from langchain_community.embeddings import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
embeddings = OpenAIEmbeddings()
                text_embeddings = embeddings.embed_documents(texts)
                text_embedding_pairs = list(zip(texts, text_embeddings))
                db = Annoy.from_embeddings(text_embedding_pairs, embeddings)
        """
        texts = [t[0] for t in text_embeddings]
        embeddings = [t[1] for t in text_embeddings]
        return cls.__from(
            texts, embeddings, embedding, metadatas, metric, trees, n_jobs, **kwargs
        )
[docs]    def save_local(self, folder_path: str, prefault: bool = False) -> None:
        """Save Annoy index, docstore, and index_to_docstore_id to disk.
        Args:
            folder_path: folder path to save index, docstore,
                and index_to_docstore_id to.
            prefault: Whether to pre-load the index into memory.
        """
        path = Path(folder_path)
        os.makedirs(path, exist_ok=True)
        # save index, index config, docstore and index_to_docstore_id
        config_object = ConfigParser()
        config_object["ANNOY"] = {
            "f": self.index.f,
            "metric": self.metric,
        }
        self.index.save(str(path / "index.annoy"), prefault=prefault)
        with open(path / "index.pkl", "wb") as file:
            pickle.dump((self.docstore, self.index_to_docstore_id, config_object), file)
[docs]    @classmethod
    def load_local(
        cls,
        folder_path: str,
        embeddings: Embeddings,
    ) -> Annoy:
        """Load Annoy index, docstore, and index_to_docstore_id to disk.
        Args:
Args:
            folder_path: folder path to load index, docstore,
                and index_to_docstore_id from.
            embeddings: Embeddings to use when generating queries.
        """
        path = Path(folder_path)
        # load index separately since it is not picklable
        annoy = dependable_annoy_import()
        # load docstore and index_to_docstore_id
        with open(path / "index.pkl", "rb") as file:
            docstore, index_to_docstore_id, config_object = pickle.load(file)
        f = int(config_object["ANNOY"]["f"])
        metric = config_object["ANNOY"]["metric"]
        index = annoy.AnnoyIndex(f, metric=metric)
        index.load(str(path / "index.annoy"))
        return cls(
            embeddings.embed_query, index, metric, docstore, index_to_docstore_id
        )
Source code for langchain_community.vectorstores.vespa
from __future__ import annotations
from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore, VectorStoreRetriever
[docs]class VespaStore(VectorStore):
    """
    `Vespa` vector store.
    To use, you should have the python client library ``pyvespa`` installed.
    Example:
        .. code-block:: python
            from langchain_community.vectorstores import VespaStore
            from langchain_community.embeddings.openai import OpenAIEmbeddings
            from vespa.application import Vespa
            # Create a vespa client dependent upon your application,
            # e.g. either connecting to Vespa Cloud or a local deployment
            # such as Docker. Please refer to the PyVespa documentation on
            # how to initialize the client.
            vespa_app = Vespa(url="...", port=..., application_package=...)
            # You need to instruct LangChain on which fields to use for embeddings
            vespa_config = dict(
                page_content_field="text",
                embedding_field="embedding",
                input_field="query_embedding",
                metadata_fields=["date", "rating", "author"]
            )
            embedding_function = OpenAIEmbeddings()
            vectorstore = VespaStore(vespa_app, embedding_function, **vespa_config)
    """
[docs]    def __init__(
        self,
        app: Any,
        embedding_function: Optional[Embeddings] = None,
        page_content_field: Optional[str] = None,
        embedding_field: Optional[str] = None,
        input_field: Optional[str] = None,
input_field: Optional[str] = None,
        metadata_fields: Optional[List[str]] = None,
    ) -> None:
        """
        Initialize with a PyVespa client.
        """
        try:
            from vespa.application import Vespa
        except ImportError:
            raise ImportError(
                "Could not import Vespa python package. "
                "Please install it with `pip install pyvespa`."
            )
        if not isinstance(app, Vespa):
            raise ValueError(
                f"app should be an instance of vespa.application.Vespa, got {type(app)}"
            )
        self._vespa_app = app
        self._embedding_function = embedding_function
        self._page_content_field = page_content_field
        self._embedding_field = embedding_field
        self._input_field = input_field
        self._metadata_fields = metadata_fields
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """
        Add texts to the vectorstore.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of ids associated with the texts.
            kwargs: vectorstore specific parameters
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        embeddings = None
        if self._embedding_function is not None:
            embeddings = self._embedding_function.embed_documents(list(texts))
        if ids is None:
if ids is None:
            ids = [str(f"{i+1}") for i, _ in enumerate(texts)]
        batch = []
        for i, text in enumerate(texts):
            fields: Dict[str, Union[str, List[float]]] = {}
            if self._page_content_field is not None:
                fields[self._page_content_field] = text
            if self._embedding_field is not None and embeddings is not None:
                fields[self._embedding_field] = embeddings[i]
            if metadatas is not None and self._metadata_fields is not None:
                for metadata_field in self._metadata_fields:
                    if metadata_field in metadatas[i]:
                        fields[metadata_field] = metadatas[i][metadata_field]
            batch.append({"id": ids[i], "fields": fields})
        results = self._vespa_app.feed_batch(batch)
        for result in results:
            if not (str(result.status_code).startswith("2")):
                raise RuntimeError(
                    f"Could not add document to Vespa. "
                    f"Error code: {result.status_code}. "
                    f"Message: {result.json['message']}"
                )
        return ids
[docs]    def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]:
        if ids is None:
            return False
        batch = [{"id": id} for id in ids]
        result = self._vespa_app.delete_batch(batch)
        return sum([0 if r.status_code == 200 else 1 for r in result]) == 0
    def _create_query(
        self, query_embedding: List[float], k: int = 4, **kwargs: Any
    ) -> Dict:
        hits = k
) -> Dict:
        hits = k
        doc_embedding_field = self._embedding_field
        input_embedding_field = self._input_field
        ranking_function = kwargs["ranking"] if "ranking" in kwargs else "default"
        filter = kwargs["filter"] if "filter" in kwargs else None
        approximate = kwargs["approximate"] if "approximate" in kwargs else False
        approximate = "true" if approximate else "false"
        yql = "select * from sources * where "
        yql += f"{{targetHits: {hits}, approximate: {approximate}}}"
        yql += f"nearestNeighbor({doc_embedding_field}, {input_embedding_field})"
        if filter is not None:
            yql += f" and {filter}"
        query = {
            "yql": yql,
            f"input.query({input_embedding_field})": query_embedding,
            "ranking": ranking_function,
            "hits": hits,
        }
        return query
[docs]    def similarity_search_by_vector_with_score(
        self, query_embedding: List[float], k: int = 4, **kwargs: Any
    ) -> List[Tuple[Document, float]]:
        """
        Performs similarity search from a embeddings vector.
        Args:
            query_embedding: Embeddings vector to search for.
            k: Number of results to return.
            custom_query: Use this custom query instead default query (kwargs)
            kwargs: other vector store specific parameters
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        if "custom_query" in kwargs:
            query = kwargs["custom_query"]
        else:
            query = self._create_query(query_embedding, k, **kwargs)
        try:
query = self._create_query(query_embedding, k, **kwargs)
        try:
            response = self._vespa_app.query(body=query)
        except Exception as e:
            raise RuntimeError(
                f"Could not retrieve data from Vespa: "
                f"{e.args[0][0]['summary']}. "
                f"Error: {e.args[0][0]['message']}"
            )
        if not str(response.status_code).startswith("2"):
            raise RuntimeError(
                f"Could not retrieve data from Vespa. "
                f"Error code: {response.status_code}. "
                f"Message: {response.json['message']}"
            )
        root = response.json["root"]
        if "errors" in root:
            import json
            raise RuntimeError(json.dumps(root["errors"]))
        if response is None or response.hits is None:
            return []
        docs = []
        for child in response.hits:
            page_content = child["fields"][self._page_content_field]
            score = child["relevance"]
            metadata = {"id": child["id"]}
            if self._metadata_fields is not None:
                for field in self._metadata_fields:
                    metadata[field] = child["fields"].get(field)
            doc = Document(page_content=page_content, metadata=metadata)
            docs.append((doc, score))
        return docs
[docs]    def similarity_search_by_vector(
        self, embedding: List[float], k: int = 4, **kwargs: Any
    ) -> List[Document]:
        results = self.similarity_search_by_vector_with_score(embedding, k, **kwargs)
        return [r[0] for r in results]
[docs]    def similarity_search_with_score(
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Tuple[Document, float]]:
        query_emb = []
        if self._embedding_function is not None:
            query_emb = self._embedding_function.embed_query(query)
        return self.similarity_search_by_vector_with_score(query_emb, k, **kwargs)
[docs]    def similarity_search(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Document]:
        results = self.similarity_search_with_score(query, k, **kwargs)
        return [r[0] for r in results]
[docs]    def max_marginal_relevance_search(
        self,
        query: str,
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        raise NotImplementedError("MMR search not implemented")
[docs]    def max_marginal_relevance_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        fetch_k: int = 20,
        lambda_mult: float = 0.5,
        **kwargs: Any,
    ) -> List[Document]:
        raise NotImplementedError("MMR search by vector not implemented")
[docs]    @classmethod
    def from_texts(
        cls: Type[VespaStore],
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> VespaStore:
        vespa = cls(embedding_function=embedding, **kwargs)
        vespa.add_texts(texts=texts, metadatas=metadatas, ids=ids)
        return vespa
[docs]    def as_retriever(self, **kwargs: Any) -> VectorStoreRetriever:
        return super().as_retriever(**kwargs)
Source code for langchain_community.vectorstores.sqlitevss
from __future__ import annotations
import json
import logging
import warnings
from typing import (
    TYPE_CHECKING,
    Any,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
)
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore
if TYPE_CHECKING:
    import sqlite3
logger = logging.getLogger(__name__)
[docs]class SQLiteVSS(VectorStore):
    """Wrapper around SQLite with vss extension as a vector database.
    To use, you should have the ``sqlite-vss`` python package installed.
    Example:
        .. code-block:: python
            from langchain_community.vectorstores import SQLiteVSS
            from langchain_community.embeddings.openai import OpenAIEmbeddings
            ...
    """
[docs]    def __init__(
        self,
        table: str,
        connection: Optional[sqlite3.Connection],
        embedding: Embeddings,
        db_file: str = "vss.db",
    ):
        """Initialize with sqlite client with vss extension."""
        try:
            import sqlite_vss  # noqa  # pylint: disable=unused-import
        except ImportError:
            raise ImportError(
                "Could not import sqlite-vss python package. "
                "Please install it with `pip install sqlite-vss`."
            )
        if not connection:
            connection = self.create_connection(db_file)
        if not isinstance(embedding, Embeddings):
            warnings.warn("embeddings input must be Embeddings object.")
        self._connection = connection
        self._table = table
        self._embedding = embedding
        self.create_table_if_not_exists()