viruPVS/langchain-docs huggingface dataset:
assert self.config
        assert self.config.host and self.config.port
        assert (
            self.config.column_map
            and self.config.database
            and self.config.table
            and self.config.metric
        )
        for k in ["id", "embedding", "document", "metadata", "uuid"]:
            assert k in self.config.column_map
        assert self.config.metric in [
            "angular",
            "euclidean",
            "manhattan",
            "hamming",
            "dot",
        ]
        # initialize the schema
        dim = len(embedding.embed_query("test"))
        index_params = (
            (
                ",".join([f"'{k}={v}'" for k, v in self.config.index_param.items()])
                if self.config.index_param
                else ""
            )
            if isinstance(self.config.index_param, Dict)
            else ",".join([str(p) for p in self.config.index_param])
            if isinstance(self.config.index_param, List)
            else self.config.index_param
        )
        self.schema = f"""\
CREATE TABLE IF NOT EXISTS {self.config.database}.{self.config.table}(
    {self.config.column_map['id']} Nullable(String),
    {self.config.column_map['document']} Nullable(String),
    {self.config.column_map['embedding']} Array(Float32),
    {self.config.column_map['metadata']} JSON,
    {self.config.column_map['uuid']} UUID DEFAULT generateUUIDv4(),
    CONSTRAINT cons_vec_len CHECK length({self.config.column_map['embedding']}) = {dim},
    INDEX vec_idx {self.config.column_map['embedding']} TYPE \
{self.config.index_type}({index_params}) GRANULARITY 1000
) ENGINE = MergeTree ORDER BY uuid SETTINGS index_granularity = 8192\
"""
"""
        self.dim = dim
        self.BS = "\\"
        self.must_escape = ("\\", "'")
        self.embedding_function = embedding
        self.dist_order = "ASC"  # Only support ConsingDistance and L2Distance
        # Create a connection to clickhouse
        self.client = get_client(
            host=self.config.host,
            port=self.config.port,
            username=self.config.username,
            password=self.config.password,
            **kwargs,
        )
        # Enable JSON type
        self.client.command("SET allow_experimental_object_type=1")
        # Enable Annoy index
        self.client.command("SET allow_experimental_annoy_index=1")
        self.client.command(self.schema)
    @property
    def embeddings(self) -> Embeddings:
        return self.embedding_function
[docs]    def escape_str(self, value: str) -> str:
        return "".join(f"{self.BS}{c}" if c in self.must_escape else c for c in value)
    def _build_insert_sql(self, transac: Iterable, column_names: Iterable[str]) -> str:
        ks = ",".join(column_names)
        _data = []
        for n in transac:
            n = ",".join([f"'{self.escape_str(str(_n))}'" for _n in n])
            _data.append(f"({n})")
        i_str = f"""
                INSERT INTO TABLE 
                    {self.config.database}.{self.config.table}({ks})
                VALUES
                {','.join(_data)}
                """
        return i_str
    def _insert(self, transac: Iterable, column_names: Iterable[str]) -> None:
        _insert_query = self._build_insert_sql(transac, column_names)
        self.client.command(_insert_query)
self.client.command(_insert_query)
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        batch_size: int = 32,
        ids: Optional[Iterable[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Insert more texts through the embeddings and add to the VectorStore.
        Args:
            texts: Iterable of strings to add to the VectorStore.
            ids: Optional list of ids to associate with the texts.
            batch_size: Batch size of insertion
            metadata: Optional column data to be inserted
        Returns:
            List of ids from adding the texts into the VectorStore.
        """
        # Embed and create the documents
        ids = ids or [sha1(t.encode("utf-8")).hexdigest() for t in texts]
        colmap_ = self.config.column_map
        transac = []
        column_names = {
            colmap_["id"]: ids,
            colmap_["document"]: texts,
            colmap_["embedding"]: self.embedding_function.embed_documents(list(texts)),
        }
        metadatas = metadatas or [{} for _ in texts]
        column_names[colmap_["metadata"]] = map(json.dumps, metadatas)
        assert len(set(colmap_) - set(column_names)) >= 0
        keys, values = zip(*column_names.items())
        try:
            t = None
            for v in self.pgbar(
                zip(*values), desc="Inserting data...", total=len(metadatas)
            ):
                assert (
                    len(v[keys.index(self.config.column_map["embedding"])]) == self.dim
                )
                transac.append(v)
)
                transac.append(v)
                if len(transac) == batch_size:
                    if t:
                        t.join()
                    t = Thread(target=self._insert, args=[transac, keys])
                    t.start()
                    transac = []
            if len(transac) > 0:
                if t:
                    t.join()
                self._insert(transac, keys)
            return [i for i in ids]
        except Exception as e:
            logger.error(f"\033[91m\033[1m{type(e)}\033[0m \033[95m{str(e)}\033[0m")
            return []
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[Dict[Any, Any]]] = None,
        config: Optional[ClickhouseSettings] = None,
        text_ids: Optional[Iterable[str]] = None,
        batch_size: int = 32,
        **kwargs: Any,
    ) -> Clickhouse:
        """Create ClickHouse wrapper with existing texts
        Args:
            embedding_function (Embeddings): Function to extract text embedding
            texts (Iterable[str]): List or tuple of strings to be added
            config (ClickHouseSettings, Optional): ClickHouse configuration
            text_ids (Optional[Iterable], optional): IDs for the texts.
                                                     Defaults to None.
            batch_size (int, optional): Batchsize when transmitting data to ClickHouse.
                                        Defaults to 32.
            metadata (List[dict], optional): metadata to texts. Defaults to None.
            Other keyword arguments will pass into
Other keyword arguments will pass into
                [clickhouse-connect](https://clickhouse.com/docs/en/integrations/python#clickhouse-connect-driver-api)
        Returns:
            ClickHouse Index
        """
        ctx = cls(embedding, config, **kwargs)
        ctx.add_texts(texts, ids=text_ids, batch_size=batch_size, metadatas=metadatas)
        return ctx
    def __repr__(self) -> str:
        """Text representation for ClickHouse Vector Store, prints backends, username
            and schemas. Easy to use with `str(ClickHouse())`
        Returns:
            repr: string to show connection info and data schema
        """
        _repr = f"\033[92m\033[1m{self.config.database}.{self.config.table} @ "
        _repr += f"{self.config.host}:{self.config.port}\033[0m\n\n"
        _repr += f"\033[1musername: {self.config.username}\033[0m\n\nTable Schema:\n"
        _repr += "-" * 51 + "\n"
        for r in self.client.query(
            f"DESC {self.config.database}.{self.config.table}"
        ).named_results():
            _repr += (
                f"|\033[94m{r['name']:24s}\033[0m|\033[96m{r['type']:24s}\033[0m|\n"
            )
        _repr += "-" * 51 + "\n"
        return _repr
    def _build_query_sql(
        self, q_emb: List[float], topk: int, where_str: Optional[str] = None
    ) -> str:
        q_emb_str = ",".join(map(str, q_emb))
        if where_str:
if where_str:
            where_str = f"PREWHERE {where_str}"
        else:
            where_str = ""
        settings_strs = []
        if self.config.index_query_params:
            for k in self.config.index_query_params:
                settings_strs.append(f"SETTING {k}={self.config.index_query_params[k]}")
        q_str = f"""
            SELECT {self.config.column_map['document']}, 
                {self.config.column_map['metadata']}, dist
            FROM {self.config.database}.{self.config.table}
            {where_str}
            ORDER BY L2Distance({self.config.column_map['embedding']}, [{q_emb_str}]) 
                AS dist {self.dist_order}
            LIMIT {topk} {' '.join(settings_strs)}
            """
        return q_str
[docs]    def similarity_search(
        self, query: str, k: int = 4, where_str: Optional[str] = None, **kwargs: Any
    ) -> List[Document]:
        """Perform a similarity search with ClickHouse
        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.
            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.
        Returns:
            List[Document]: List of Documents
        """
        return self.similarity_search_by_vector(
            self.embedding_function.embed_query(query), k, where_str, **kwargs
        )
self.embedding_function.embed_query(query), k, where_str, **kwargs
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        where_str: Optional[str] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Perform a similarity search with ClickHouse by vectors
        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.
            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.
        Returns:
            List[Document]: List of documents
        """
        q_str = self._build_query_sql(embedding, k, where_str)
        try:
            return [
                Document(
                    page_content=r[self.config.column_map["document"]],
                    metadata=r[self.config.column_map["metadata"]],
                )
                for r in self.client.query(q_str).named_results()
            ]
        except Exception as e:
            logger.error(f"\033[91m\033[1m{type(e)}\033[0m \033[95m{str(e)}\033[0m")
            return []
[docs]    def similarity_search_with_relevance_scores(
        self, query: str, k: int = 4, where_str: Optional[str] = None, **kwargs: Any
    ) -> List[Tuple[Document, float]]:
) -> List[Tuple[Document, float]]:
        """Perform a similarity search with ClickHouse
        Args:
            query (str): query string
            k (int, optional): Top K neighbors to retrieve. Defaults to 4.
            where_str (Optional[str], optional): where condition string.
                                                 Defaults to None.
            NOTE: Please do not let end-user to fill this and always be aware
                  of SQL injection. When dealing with metadatas, remember to
                  use `{self.metadata_column}.attribute` instead of `attribute`
                  alone. The default name for it is `metadata`.
        Returns:
            List[Document]: List of (Document, similarity)
        """
        q_str = self._build_query_sql(
            self.embedding_function.embed_query(query), k, where_str
        )
        try:
            return [
                (
                    Document(
                        page_content=r[self.config.column_map["document"]],
                        metadata=r[self.config.column_map["metadata"]],
                    ),
                    r["dist"],
                )
                for r in self.client.query(q_str).named_results()
            ]
        except Exception as e:
            logger.error(f"\033[91m\033[1m{type(e)}\033[0m \033[95m{str(e)}\033[0m")
            return []
[docs]    def drop(self) -> None:
        """
        Helper function: Drop data
        """
        self.client.command(
            f"DROP TABLE IF EXISTS {self.config.database}.{self.config.table}"
        )
    @property
    def metadata_column(self) -> str:
        return self.config.column_map["metadata"]
Source code for langchain_community.vectorstores.deeplake
from __future__ import annotations
import logging
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
import numpy as np
try:
    import deeplake
    from deeplake import VectorStore as DeepLakeVectorStore
    from deeplake.core.fast_forwarding import version_compare
    _DEEPLAKE_INSTALLED = True
except ImportError:
    _DEEPLAKE_INSTALLED = False
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore
from langchain_community.vectorstores.utils import maximal_marginal_relevance
logger = logging.getLogger(__name__)
[docs]class DeepLake(VectorStore):
    """`Activeloop Deep Lake` vector store.
    We integrated deeplake's similarity search and filtering for fast prototyping.
    Now, it supports Tensor Query Language (TQL) for production use cases
    over billion rows.
    Why Deep Lake?
    - Not only stores embeddings, but also the original data with version control.
    - Serverless, doesn't require another service and can be used with major
        cloud providers (S3, GCS, etc.)
    - More than just a multi-modal vector store. You can use the dataset
        to fine-tune your own LLM models.
    To use, you should have the ``deeplake`` python package installed.
    Example:
        .. code-block:: python
                from langchain_community.vectorstores import DeepLake
                from langchain_community.embeddings.openai import OpenAIEmbeddings
                embeddings = OpenAIEmbeddings()
                vectorstore = DeepLake("langchain_store", embeddings.embed_query)
    """
vectorstore = DeepLake("langchain_store", embeddings.embed_query)
    """
    _LANGCHAIN_DEFAULT_DEEPLAKE_PATH = "./deeplake/"
[docs]    def __init__(
        self,
        dataset_path: str = _LANGCHAIN_DEFAULT_DEEPLAKE_PATH,
        token: Optional[str] = None,
        embedding: Optional[Embeddings] = None,
        embedding_function: Optional[Embeddings] = None,
        read_only: bool = False,
        ingestion_batch_size: int = 1000,
        num_workers: int = 0,
        verbose: bool = True,
        exec_option: Optional[str] = None,
        runtime: Optional[Dict] = None,
        index_params: Optional[Dict[str, Union[int, str]]] = None,
        **kwargs: Any,
    ) -> None:
        """Creates an empty DeepLakeVectorStore or loads an existing one.
        The DeepLakeVectorStore is located at the specified ``path``.
        Examples:
            >>> # Create a vector store with default tensors
            >>> deeplake_vectorstore = DeepLake(
            ...        path = <path_for_storing_Data>,
            ... )
            >>>
            >>> # Create a vector store in the Deep Lake Managed Tensor Database
            >>> data = DeepLake(
            ...        path = "hub://org_id/dataset_name",
            ...        runtime = {"tensor_db": True},
            ... )
        Args:
            dataset_path (str): Path to existing dataset or where to create
                a new one. Defaults to _LANGCHAIN_DEFAULT_DEEPLAKE_PATH.
            token (str, optional):  Activeloop token, for fetching credentials
                to the dataset at path if it is a Deep Lake dataset.
to the dataset at path if it is a Deep Lake dataset.
                Tokens are normally autogenerated. Optional.
            embedding (Embeddings, optional): Function to convert
                either documents or query. Optional.
            embedding_function (Embeddings, optional): Function to convert
                either documents or query. Optional. Deprecated: keeping this
                parameter for backwards compatibility.
            read_only (bool): Open dataset in read-only mode. Default is False.
            ingestion_batch_size (int): During data ingestion, data is divided
                into batches. Batch size is the size of each batch.
                Default is 1000.
            num_workers (int): Number of workers to use during data ingestion.
                Default is 0.
            verbose (bool): Print dataset summary after each operation.
                Default is True.
            exec_option (str, optional): DeepLakeVectorStore supports 3 ways to perform
                searching - "python", "compute_engine", "tensor_db" and auto.
                Default is None.
                - ``auto``- Selects the best execution method based on the storage
                    location of the Vector Store. It is the default option.
                - ``python`` - Pure-python implementation that runs on the client.
                    WARNING: using this with big datasets can lead to memory
                    issues. Data can be stored anywhere.
                - ``compute_engine`` - C++ implementation of the Deep Lake Compute
                    Engine that runs on the client. Can be used for any data stored in
                    or connected to Deep Lake. Not for in-memory or local datasets.
                - ``tensor_db`` - Hosted Managed Tensor Database that is
                    responsible for storage and query execution. Only for data stored in
                    the Deep Lake Managed Database. Use runtime = {"db_engine": True}
                    during dataset creation.
during dataset creation.
            runtime (Dict, optional): Parameters for creating the Vector Store in
                Deep Lake's Managed Tensor Database. Not applicable when loading an
                existing Vector Store. To create a Vector Store in the Managed Tensor
                Database, set `runtime = {"tensor_db": True}`.
            index_params (Optional[Dict[str, Union[int, str]]], optional): Dictionary
                containing information about vector index that will be created. Defaults
                to None, which will utilize ``DEFAULT_VECTORSTORE_INDEX_PARAMS`` from
                ``deeplake.constants``. The specified key-values override the default
                ones.
                - threshold: The threshold for the dataset size above which an index
                    will be created for the embedding tensor. When the threshold value
                    is set to -1, index creation is turned off. Defaults to -1, which
                    turns off the index.
                - distance_metric: This key specifies the method of calculating the
                    distance between vectors when creating the vector database (VDB)
                    index. It can either be a string that corresponds to a member of
                    the DistanceType enumeration, or the string value itself.
                    - If no value is provided, it defaults to "L2".
                    - "L2" corresponds to DistanceType.L2_NORM.
                    - "COS" corresponds to DistanceType.COSINE_SIMILARITY.
                - additional_params: Additional parameters for fine-tuning the index.
            **kwargs: Other optional keyword arguments.
        Raises:
            ValueError: If some condition is not met.
        """
        self.ingestion_batch_size = ingestion_batch_size
        self.num_workers = num_workers
        self.verbose = verbose
        if _DEEPLAKE_INSTALLED is False:
            raise ImportError(
                "Could not import deeplake python package. "
raise ImportError(
                "Could not import deeplake python package. "
                "Please install it with `pip install deeplake[enterprise]`."
            )
        if (
            runtime == {"tensor_db": True}
            and version_compare(deeplake.__version__, "3.6.7") == -1
        ):
            raise ImportError(
                "To use tensor_db option you need to update deeplake to `3.6.7` or "
                "higher. "
                f"Currently installed deeplake version is {deeplake.__version__}. "
            )
        self.dataset_path = dataset_path
        if embedding_function:
            logger.warning(
                "Using embedding function is deprecated and will be removed "
                "in the future. Please use embedding instead."
            )
        self.vectorstore = DeepLakeVectorStore(
            path=self.dataset_path,
            embedding_function=embedding_function or embedding,
            read_only=read_only,
            token=token,
            exec_option=exec_option,
            verbose=verbose,
            runtime=runtime,
            index_params=index_params,
            **kwargs,
        )
        self._embedding_function = embedding_function or embedding
        self._id_tensor_name = "ids" if "ids" in self.vectorstore.tensors() else "id"
    @property
    def embeddings(self) -> Optional[Embeddings]:
        return self._embedding_function
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embeddings and add to the vectorstore.
        Examples:
"""Run more texts through the embeddings and add to the vectorstore.
        Examples:
            >>> ids = deeplake_vectorstore.add_texts(
            ...     texts = <list_of_texts>,
            ...     metadatas = <list_of_metadata_jsons>,
            ...     ids = <list_of_ids>,
            ... )
        Args:
            texts (Iterable[str]): Texts to add to the vectorstore.
            metadatas (Optional[List[dict]], optional): Optional list of metadatas.
            ids (Optional[List[str]], optional): Optional list of IDs.
            embedding_function (Optional[Embeddings], optional): Embedding function
                to use to convert the text into embeddings.
            **kwargs (Any): Any additional keyword arguments passed is not supported
                by this method.
        Returns:
            List[str]: List of IDs of the added texts.
        """
        if kwargs:
            unsupported_items = "`, `".join(set(kwargs.keys()))
            raise TypeError(
                f"`{unsupported_items}` is/are not a valid argument to add_text method"
            )
        kwargs = {}
        if ids:
            if self._id_tensor_name == "ids":  # for backwards compatibility
                kwargs["ids"] = ids
            else:
                kwargs["id"] = ids
        if metadatas is None:
            metadatas = [{}] * len(list(texts))
        if not isinstance(texts, list):
            texts = list(texts)
        if texts is None:
            raise ValueError("`texts` parameter shouldn't be None.")
        elif len(texts) == 0:
            raise ValueError("`texts` parameter shouldn't be empty.")
        return self.vectorstore.add(
            text=texts,
            metadata=metadatas,