viruPVS/langchain-docs huggingface dataset:
Source code for langchain_community.vectorstores.zilliz
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from langchain_core.embeddings import Embeddings
from langchain_community.vectorstores.milvus import Milvus
logger = logging.getLogger(__name__)
[docs]class Zilliz(Milvus):
    """`Zilliz` vector store.
    You need to have `pymilvus` installed and a
    running Zilliz database.
    See the following documentation for how to run a Zilliz instance:
    https://docs.zilliz.com/docs/create-cluster
    IF USING L2/IP metric IT IS HIGHLY SUGGESTED TO NORMALIZE YOUR DATA.
    Args:
        embedding_function (Embeddings): Function used to embed the text.
        collection_name (str): Which Zilliz collection to use. Defaults to
            "LangChainCollection".
        connection_args (Optional[dict[str, any]]): The connection args used for
            this class comes in the form of a dict.
        consistency_level (str): The consistency level to use for a collection.
            Defaults to "Session".
        index_params (Optional[dict]): Which index params to use. Defaults to
            HNSW/AUTOINDEX depending on service.
        search_params (Optional[dict]): Which search params to use. Defaults to
            default of index.
        drop_old (Optional[bool]): Whether to drop the current collection. Defaults
            to False.
    The connection args used for this class comes in the form of a dict,
    here are a few of the options:
        address (str): The actual address of Zilliz
            instance. Example address: "localhost:19530"
instance. Example address: "localhost:19530"
        uri (str): The uri of Zilliz instance. Example uri:
            "https://in03-ba4234asae.api.gcp-us-west1.zillizcloud.com",
        host (str): The host of Zilliz instance. Default at "localhost",
            PyMilvus will fill in the default host if only port is provided.
        port (str/int): The port of Zilliz instance. Default at 19530, PyMilvus
            will fill in the default port if only host is provided.
        user (str): Use which user to connect to Zilliz instance. If user and
            password are provided, we will add related header in every RPC call.
        password (str): Required when user is provided. The password
            corresponding to the user.
        token (str): API key, for serverless clusters which can be used as
            replacements for user and password.
        secure (bool): Default is false. If set to true, tls will be enabled.
        client_key_path (str): If use tls two-way authentication, need to
            write the client.key path.
        client_pem_path (str): If use tls two-way authentication, need to
            write the client.pem path.
        ca_pem_path (str): If use tls two-way authentication, need to write
            the ca.pem path.
        server_pem_path (str): If use tls one-way authentication, need to
            write the server.pem path.
        server_name (str): If use tls, need to write the common name.
    Example:
        .. code-block:: python
        from langchain_community.vectorstores import Zilliz
        from langchain_community.embeddings import OpenAIEmbeddings
        embedding = OpenAIEmbeddings()
embedding = OpenAIEmbeddings()
        # Connect to a Zilliz instance
        milvus_store = Milvus(
            embedding_function = embedding,
            collection_name = "LangChainCollection",
            connection_args = {
                "uri": "https://in03-ba4234asae.api.gcp-us-west1.zillizcloud.com",
                "user": "temp",
                "password": "temp",
                "token": "temp", # API key as replacements for user and password
                "secure": True
            }
            drop_old: True,
        )
    Raises:
        ValueError: If the pymilvus python package is not installed.
    """
    def _create_index(self) -> None:
        """Create a index on the collection"""
        from pymilvus import Collection, MilvusException
        if isinstance(self.col, Collection) and self._get_index() is None:
            try:
                # If no index params, use a default AutoIndex based one
                if self.index_params is None:
                    self.index_params = {
                        "metric_type": "L2",
                        "index_type": "AUTOINDEX",
                        "params": {},
                    }
                try:
                    self.col.create_index(
                        self._vector_field,
                        index_params=self.index_params,
                        using=self.alias,
                    )
                # If default did not work, most likely Milvus self-hosted
                except MilvusException:
                    # Use HNSW based index
                    self.index_params = {
                        "metric_type": "L2",
                        "index_type": "HNSW",
                        "params": {"M": 8, "efConstruction": 64},
                    }
                    self.col.create_index(
}
                    self.col.create_index(
                        self._vector_field,
                        index_params=self.index_params,
                        using=self.alias,
                    )
                logger.debug(
                    "Successfully created an index on collection: %s",
                    self.collection_name,
                )
            except MilvusException as e:
                logger.error(
                    "Failed to create an index on collection: %s", self.collection_name
                )
                raise e
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        collection_name: str = "LangChainCollection",
        connection_args: Optional[Dict[str, Any]] = None,
        consistency_level: str = "Session",
        index_params: Optional[dict] = None,
        search_params: Optional[dict] = None,
        drop_old: bool = False,
        **kwargs: Any,
    ) -> Zilliz:
        """Create a Zilliz collection, indexes it with HNSW, and insert data.
        Args:
            texts (List[str]): Text data.
            embedding (Embeddings): Embedding function.
            metadatas (Optional[List[dict]]): Metadata for each text if it exists.
                Defaults to None.
            collection_name (str, optional): Collection name to use. Defaults to
                "LangChainCollection".
            connection_args (dict[str, Any], optional): Connection args to use. Defaults
                to DEFAULT_MILVUS_CONNECTION.
            consistency_level (str, optional): Which consistency level to use. Defaults
                to "Session".
            index_params (Optional[dict], optional): Which index_params to use.
                Defaults to None.
Defaults to None.
            search_params (Optional[dict], optional): Which search params to use.
                Defaults to None.
            drop_old (Optional[bool], optional): Whether to drop the collection with
                that name if it exists. Defaults to False.
        Returns:
            Zilliz: Zilliz Vector Store
        """
        vector_db = cls(
            embedding_function=embedding,
            collection_name=collection_name,
            connection_args=connection_args or {},
            consistency_level=consistency_level,
            index_params=index_params,
            search_params=search_params,
            drop_old=drop_old,
            **kwargs,
        )
        vector_db.add_texts(texts=texts, metadatas=metadatas)
        return vector_db
Source code for langchain_community.vectorstores.typesense
from __future__ import annotations
import uuid
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.utils import get_from_env
from langchain_core.vectorstores import VectorStore
if TYPE_CHECKING:
    from typesense.client import Client
    from typesense.collection import Collection
[docs]class Typesense(VectorStore):
    """`Typesense` vector store.
    To use, you should have the ``typesense`` python package installed.
    Example:
        .. code-block:: python
            from langchain_community.embedding.openai import OpenAIEmbeddings
            from langchain_community.vectorstores import Typesense
            import typesense
            node = {
                "host": "localhost",  # For Typesense Cloud use xxx.a1.typesense.net
                "port": "8108",       # For Typesense Cloud use 443
                "protocol": "http"    # For Typesense Cloud use https
            }
            typesense_client = typesense.Client(
                {
                  "nodes": [node],
                  "api_key": "<API_KEY>",
                  "connection_timeout_seconds": 2
                }
            )
            typesense_collection_name = "langchain-memory"
            embedding = OpenAIEmbeddings()
            vectorstore = Typesense(
                typesense_client=typesense_client,
                embedding=embedding,
                typesense_collection_name=typesense_collection_name,
                text_key="text",
            )
    """
[docs]    def __init__(
        self,
        typesense_client: Client,
        embedding: Embeddings,
        *,
        typesense_collection_name: Optional[str] = None,
*,
        typesense_collection_name: Optional[str] = None,
        text_key: str = "text",
    ):
        """Initialize with Typesense client."""
        try:
            from typesense import Client
        except ImportError:
            raise ImportError(
                "Could not import typesense python package. "
                "Please install it with `pip install typesense`."
            )
        if not isinstance(typesense_client, Client):
            raise ValueError(
                f"typesense_client should be an instance of typesense.Client, "
                f"got {type(typesense_client)}"
            )
        self._typesense_client = typesense_client
        self._embedding = embedding
        self._typesense_collection_name = (
            typesense_collection_name or f"langchain-{str(uuid.uuid4())}"
        )
        self._text_key = text_key
    @property
    def _collection(self) -> Collection:
        return self._typesense_client.collections[self._typesense_collection_name]
    @property
    def embeddings(self) -> Embeddings:
        return self._embedding
    def _prep_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]],
        ids: Optional[List[str]],
    ) -> List[dict]:
        """Embed and create the documents"""
        _ids = ids or (str(uuid.uuid4()) for _ in texts)
        _metadatas: Iterable[dict] = metadatas or ({} for _ in texts)
        embedded_texts = self._embedding.embed_documents(list(texts))
        return [
            {"id": _id, "vec": vec, f"{self._text_key}": text, "metadata": metadata}
for _id, vec, text, metadata in zip(_ids, embedded_texts, texts, _metadatas)
        ]
    def _create_collection(self, num_dim: int) -> None:
        fields = [
            {"name": "vec", "type": "float[]", "num_dim": num_dim},
            {"name": f"{self._text_key}", "type": "string"},
            {"name": ".*", "type": "auto"},
        ]
        self._typesense_client.collections.create(
            {"name": self._typesense_collection_name, "fields": fields}
        )
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Run more texts through the embedding and add to the vectorstore.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
            ids: Optional list of ids to associate with the texts.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        from typesense.exceptions import ObjectNotFound
        docs = self._prep_texts(texts, metadatas, ids)
        try:
            self._collection.documents.import_(docs, {"action": "upsert"})
        except ObjectNotFound:
            # Create the collection if it doesn't already exist
            self._create_collection(len(docs[0]["vec"]))
            self._collection.documents.import_(docs, {"action": "upsert"})
        return [doc["id"] for doc in docs]
return [doc["id"] for doc in docs]
[docs]    def similarity_search_with_score(
        self,
        query: str,
        k: int = 10,
        filter: Optional[str] = "",
    ) -> List[Tuple[Document, float]]:
        """Return typesense documents most similar to query, along with scores.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 10.
                Minimum 10 results would be returned.
            filter: typesense filter_by expression to filter documents on
        Returns:
            List of Documents most similar to the query and score for each
        """
        embedded_query = [str(x) for x in self._embedding.embed_query(query)]
        query_obj = {
            "q": "*",
            "vector_query": f'vec:([{",".join(embedded_query)}], k:{k})',
            "filter_by": filter,
            "collection": self._typesense_collection_name,
        }
        docs = []
        response = self._typesense_client.multi_search.perform(
            {"searches": [query_obj]}, {}
        )
        for hit in response["results"][0]["hits"]:
            document = hit["document"]
            metadata = document["metadata"]
            text = document[self._text_key]
            score = hit["vector_distance"]
            docs.append((Document(page_content=text, metadata=metadata), score))
        return docs
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 10,
        filter: Optional[str] = "",
        **kwargs: Any,
    ) -> List[Document]:
        """Return typesense documents most similar to query.
) -> List[Document]:
        """Return typesense documents most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 10.
                Minimum 10 results would be returned.
            filter: typesense filter_by expression to filter documents on
        Returns:
            List of Documents most similar to the query and score for each
        """
        docs_and_score = self.similarity_search_with_score(query, k=k, filter=filter)
        return [doc for doc, _ in docs_and_score]
[docs]    @classmethod
    def from_client_params(
        cls,
        embedding: Embeddings,
        *,
        host: str = "localhost",
        port: Union[str, int] = "8108",
        protocol: str = "http",
        typesense_api_key: Optional[str] = None,
        connection_timeout_seconds: int = 2,
        **kwargs: Any,
    ) -> Typesense:
        """Initialize Typesense directly from client parameters.
        Example:
            .. code-block:: python
                from langchain_community.embedding.openai import OpenAIEmbeddings
                from langchain_community.vectorstores import Typesense
                # Pass in typesense_api_key as kwarg or set env var "TYPESENSE_API_KEY".
                vectorstore = Typesense(
                    OpenAIEmbeddings(),
                    host="localhost",
                    port="8108",
                    protocol="http",
                    typesense_collection_name="langchain-memory",
                )
        """
        try:
            from typesense import Client
        except ImportError:
            raise ValueError(
                "Could not import typesense python package. "
                "Please install it with `pip install typesense`."
            )
"Please install it with `pip install typesense`."
            )
        node = {
            "host": host,
            "port": str(port),
            "protocol": protocol,
        }
        typesense_api_key = typesense_api_key or get_from_env(
            "typesense_api_key", "TYPESENSE_API_KEY"
        )
        client_config = {
            "nodes": [node],
            "api_key": typesense_api_key,
            "connection_timeout_seconds": connection_timeout_seconds,
        }
        return cls(Client(client_config), embedding, **kwargs)
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        ids: Optional[List[str]] = None,
        typesense_client: Optional[Client] = None,
        typesense_client_params: Optional[dict] = None,
        typesense_collection_name: Optional[str] = None,
        text_key: str = "text",
        **kwargs: Any,
    ) -> Typesense:
        """Construct Typesense wrapper from raw text."""
        if typesense_client:
            vectorstore = cls(typesense_client, embedding, **kwargs)
        elif typesense_client_params:
            vectorstore = cls.from_client_params(
                embedding, **typesense_client_params, **kwargs
            )
        else:
            raise ValueError(
                "Must specify one of typesense_client or typesense_client_params."
            )
        vectorstore.add_texts(texts, metadatas=metadatas, ids=ids)
        return vectorstore
Source code for langchain_community.vectorstores.clickhouse
from __future__ import annotations
import json
import logging
from hashlib import sha1
from threading import Thread
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import BaseSettings
from langchain_core.vectorstores import VectorStore
logger = logging.getLogger()
[docs]def has_mul_sub_str(s: str, *args: Any) -> bool:
    """
    Check if a string contains multiple substrings.
    Args:
        s: string to check.
        *args: substrings to check.
    Returns:
        True if all substrings are in the string, False otherwise.
    """
    for a in args:
        if a not in s:
            return False
    return True
[docs]class ClickhouseSettings(BaseSettings):
    """`ClickHouse` client configuration.
    Attribute:
        host (str) : An URL to connect to MyScale backend.
                             Defaults to 'localhost'.
        port (int) : URL port to connect with HTTP. Defaults to 8443.
        username (str) : Username to login. Defaults to None.
        password (str) : Password to login. Defaults to None.
        index_type (str): index type string.
        index_param (list): index build parameter.
        index_query_params(dict): index query parameters.
        database (str) : Database name to find the table. Defaults to 'default'.
        table (str) : Table name to operate on.
                      Defaults to 'vector_table'.
        metric (str) : Metric to compute distance,
Defaults to 'vector_table'.
        metric (str) : Metric to compute distance,
                       supported are ('angular', 'euclidean', 'manhattan', 'hamming',
                       'dot'). Defaults to 'angular'.
                       https://github.com/spotify/annoy/blob/main/src/annoymodule.cc#L149-L169
        column_map (Dict) : Column type map to project column name onto langchain
                            semantics. Must have keys: `text`, `id`, `vector`,
                            must be same size to number of columns. For example:
                            .. code-block:: python
                                {
                                    'id': 'text_id',
                                    'uuid': 'global_unique_id'
                                    'embedding': 'text_embedding',
                                    'document': 'text_plain',
                                    'metadata': 'metadata_dictionary_in_json',
                                }
                            Defaults to identity map.
    """
    host: str = "localhost"
    port: int = 8123
    username: Optional[str] = None
    password: Optional[str] = None
    index_type: str = "annoy"
    # Annoy supports L2Distance and cosineDistance.
    index_param: Optional[Union[List, Dict]] = ["'L2Distance'", 100]
    index_query_params: Dict[str, str] = {}
    column_map: Dict[str, str] = {
        "id": "id",
        "uuid": "uuid",
        "document": "document",
        "embedding": "embedding",
        "metadata": "metadata",
    }
    database: str = "default"
    table: str = "langchain"
    metric: str = "angular"
    def __getitem__(self, item: str) -> Any:
        return getattr(self, item)
    class Config:
return getattr(self, item)
    class Config:
        env_file = ".env"
        env_prefix = "clickhouse_"
        env_file_encoding = "utf-8"
[docs]class Clickhouse(VectorStore):
    """`ClickHouse VectorSearch` vector store.
    You need a `clickhouse-connect` python package, and a valid account
    to connect to ClickHouse.
    ClickHouse can not only search with simple vector indexes,
    it also supports complex query with multiple conditions,
    constraints and even sub-queries.
    For more information, please visit
        [ClickHouse official site](https://clickhouse.com/clickhouse)
    """
[docs]    def __init__(
        self,
        embedding: Embeddings,
        config: Optional[ClickhouseSettings] = None,
        **kwargs: Any,
    ) -> None:
        """ClickHouse Wrapper to LangChain
        embedding_function (Embeddings):
        config (ClickHouseSettings): Configuration to ClickHouse Client
        Other keyword arguments will pass into
            [clickhouse-connect](https://docs.clickhouse.com/)
        """
        try:
            from clickhouse_connect import get_client
        except ImportError:
            raise ImportError(
                "Could not import clickhouse connect python package. "
                "Please install it with `pip install clickhouse-connect`."
            )
        try:
            from tqdm import tqdm
            self.pgbar = tqdm
        except ImportError:
            # Just in case if tqdm is not installed
            self.pgbar = lambda x, **kwargs: x
        super().__init__()
        if config is not None:
            self.config = config
        else:
            self.config = ClickhouseSettings()
        assert self.config
        assert self.config.host and self.config.port