viruPVS/langchain-docs huggingface dataset:
)
        self.ha3_engine_client = client.Client(
            models.Config(
                endpoint=config.endpoint,
                instance_id=config.instance_id,
                protocol=config.protocol,
                access_user_name=config.username,
                access_pass_word=config.password,
            )
        )
        self.options_headers: Dict[str, str] = {}
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Insert documents into the instance..
        Args:
            texts: The text segments to be inserted into the vector storage,
             should not be empty.
            metadatas: Metadata information.
        Returns:
            id_list: List of document IDs.
        """
        def _upsert(push_doc_list: List[Dict]) -> List[str]:
            if push_doc_list is None or len(push_doc_list) == 0:
                return []
            try:
                push_request = models.PushDocumentsRequest(
                    self.options_headers, push_doc_list
                )
                push_response = self.ha3_engine_client.push_documents(
                    self.config.opt_table_name, field_name_map["id"], push_request
                )
                json_response = json.loads(push_response.body)
                if json_response["status"] == "OK":
                    return [
                        push_doc["fields"][field_name_map["id"]]
                        for push_doc in push_doc_list
                    ]
                return []
            except Exception as e:
                logger.error(
                    f"add doc to endpoint:{self.config.endpoint} "
                    f"instance_id:{self.config.instance_id} failed.",
                    e,
                )
                raise e
e,
                )
                raise e
        from alibabacloud_ha3engine_vector import models
        id_list = [sha1(t.encode("utf-8")).hexdigest() for t in texts]
        embeddings = self.embedding.embed_documents(list(texts))
        metadatas = metadatas or [{} for _ in texts]
        field_name_map = self.config.field_name_mapping
        add_doc_list = []
        text_list = list(texts)
        for idx, doc_id in enumerate(id_list):
            embedding = embeddings[idx] if idx < len(embeddings) else None
            metadata = metadatas[idx] if idx < len(metadatas) else None
            text = text_list[idx] if idx < len(text_list) else None
            add_doc: Dict[str, Any] = dict()
            add_doc_fields: Dict[str, Any] = dict()
            add_doc_fields.__setitem__(field_name_map["id"], doc_id)
            add_doc_fields.__setitem__(field_name_map["document"], text)
            if embedding is not None:
                add_doc_fields.__setitem__(
                    field_name_map["embedding"],
                    self.config.embedding_field_separator.join(
                        str(unit) for unit in embedding
                    ),
                )
            if metadata is not None:
                for md_key, md_value in metadata.items():
                    add_doc_fields.__setitem__(
                        field_name_map[md_key].split(",")[0], md_value
                    )
            add_doc.__setitem__("fields", add_doc_fields)
            add_doc.__setitem__("cmd", "add")
            add_doc_list.append(add_doc)
        return _upsert(add_doc_list)
[docs]    def similarity_search(
        self,
        query: str,
        k: int = 4,
self,
        query: str,
        k: int = 4,
        search_filter: Optional[Dict[str, Any]] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Perform similarity retrieval based on text.
        Args:
            query: Vectorize text for retrieval.，should not be empty.
            k: top n.
            search_filter: Additional filtering conditions.
        Returns:
            document_list: List of documents.
        """
        embedding = self.embedding.embed_query(query)
        return self.create_results(
            self.inner_embedding_query(
                embedding=embedding, search_filter=search_filter, k=k
            )
        )
[docs]    def similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        search_filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Perform similarity retrieval based on text with scores.
        Args:
            query: Vectorize text for retrieval.，should not be empty.
            k: top n.
            search_filter: Additional filtering conditions.
        Returns:
            document_list: List of documents.
        """
        embedding: List[float] = self.embedding.embed_query(query)
        return self.create_results_with_score(
            self.inner_embedding_query(
                embedding=embedding, search_filter=search_filter, k=k
            )
        )
[docs]    def similarity_search_by_vector(
        self,
        embedding: List[float],
        k: int = 4,
        search_filter: Optional[dict] = None,
        **kwargs: Any,
    ) -> List[Document]:
        """Perform retrieval directly using vectors.
) -> List[Document]:
        """Perform retrieval directly using vectors.
        Args:
            embedding: vectors.
            k: top n.
            search_filter: Additional filtering conditions.
        Returns:
            document_list: List of documents.
        """
        return self.create_results(
            self.inner_embedding_query(
                embedding=embedding, search_filter=search_filter, k=k
            )
        )
[docs]    def inner_embedding_query(
        self,
        embedding: List[float],
        search_filter: Optional[Dict[str, Any]] = None,
        k: int = 4,
    ) -> Dict[str, Any]:
        def generate_filter_query() -> str:
            if search_filter is None:
                return ""
            filter_clause = " AND ".join(
                [
                    create_filter(md_key, md_value)
                    for md_key, md_value in search_filter.items()
                ]
            )
            return filter_clause
        def create_filter(md_key: str, md_value: Any) -> str:
            md_filter_expr = self.config.field_name_mapping[md_key]
            if md_filter_expr is None:
                return ""
            expr = md_filter_expr.split(",")
            if len(expr) != 2:
                logger.error(
                    f"filter {md_filter_expr} express is not correct, "
                    f"must contain mapping field and operator."
                )
                return ""
            md_filter_key = expr[0].strip()
            md_filter_operator = expr[1].strip()
            if isinstance(md_value, numbers.Number):
                return f"{md_filter_key} {md_filter_operator} {md_value}"
            return f'{md_filter_key}{md_filter_operator}"{md_value}"'
return f'{md_filter_key}{md_filter_operator}"{md_value}"'
        def search_data() -> Dict[str, Any]:
            request = QueryRequest(
                table_name=self.config.table_name,
                namespace=self.config.namespace,
                vector=embedding,
                include_vector=True,
                output_fields=self.config.output_fields,
                filter=generate_filter_query(),
                top_k=k,
            )
            query_result = self.ha3_engine_client.query(request)
            return json.loads(query_result.body)
        from alibabacloud_ha3engine_vector.models import QueryRequest
        try:
            json_response = search_data()
            if (
                "errorCode" in json_response
                and "errorMsg" in json_response
                and len(json_response["errorMsg"]) > 0
            ):
                logger.error(
                    f"query {self.config.endpoint} {self.config.instance_id} "
                    f"failed:{json_response['errorMsg']}."
                )
            else:
                return json_response
        except Exception as e:
            logger.error(
                f"query instance endpoint:{self.config.endpoint} "
                f"instance_id:{self.config.instance_id} failed.",
                e,
            )
        return {}
[docs]    def create_results(self, json_result: Dict[str, Any]) -> List[Document]:
        """Assemble documents."""
        items = json_result["result"]
        query_result_list: List[Document] = []
        for item in items:
            if (
                "fields" not in item
                or self.config.field_name_mapping["document"] not in item["fields"]
            ):
                query_result_list.append(Document())
            else:
                fields = item["fields"]
                query_result_list.append(
                    Document(
fields = item["fields"]
                query_result_list.append(
                    Document(
                        page_content=fields[self.config.field_name_mapping["document"]],
                        metadata=self.create_inverse_metadata(fields),
                    )
                )
        return query_result_list
[docs]    def create_inverse_metadata(self, fields: Dict[str, Any]) -> Dict[str, Any]:
        """Create metadata from fields.
        Args:
            fields: The fields of the document. The fields must be a dict.
        Returns:
            metadata: The metadata of the document. The metadata must be a dict.
        """
        metadata: Dict[str, Any] = {}
        for key, value in fields.items():
            if key == "id" or key == "document" or key == "embedding":
                continue
            metadata[self.config.inverse_field_name_mapping[key]] = value
        return metadata
[docs]    def create_results_with_score(
        self, json_result: Dict[str, Any]
    ) -> List[Tuple[Document, float]]:
        """Parsing the returned results with scores.
        Args:
            json_result: Results from OpenSearch query.
        Returns:
            query_result_list: Results with scores.
        """
        items = json_result["result"]
        query_result_list: List[Tuple[Document, float]] = []
        for item in items:
            fields = item["fields"]
            query_result_list.append(
                (
                    Document(
                        page_content=fields[self.config.field_name_mapping["document"]],
                        metadata=self.create_inverse_metadata(fields),
                    ),
                    float(item["score"]),
                )
            )
        return query_result_list
[docs]    def delete_documents_with_texts(self, texts: List[str]) -> bool:
[docs]    def delete_documents_with_texts(self, texts: List[str]) -> bool:
        """Delete documents based on their page content.
        Args:
            texts: List of document page content.
        Returns:
           Whether the deletion was successful or not.
        """
        id_list = [sha1(t.encode("utf-8")).hexdigest() for t in texts]
        return self.delete_documents_with_document_id(id_list)
[docs]    def delete_documents_with_document_id(self, id_list: List[str]) -> bool:
        """Delete documents based on their IDs.
        Args:
            id_list: List of document IDs.
        Returns:
            Whether the deletion was successful or not.
        """
        if id_list is None or len(id_list) == 0:
            return True
        from alibabacloud_ha3engine_vector import models
        delete_doc_list = []
        for doc_id in id_list:
            delete_doc_list.append(
                {
                    "fields": {self.config.field_name_mapping["id"]: doc_id},
                    "cmd": "delete",
                }
            )
        delete_request = models.PushDocumentsRequest(
            self.options_headers, delete_doc_list
        )
        try:
            delete_response = self.ha3_engine_client.push_documents(
                self.config.opt_table_name,
                self.config.field_name_mapping["id"],
                delete_request,
            )
            json_response = json.loads(delete_response.body)
            return json_response["status"] == "OK"
        except Exception as e:
            logger.error(
                f"delete doc from :{self.config.endpoint} "
                f"instance_id:{self.config.instance_id} failed.",
                e,
            )
            raise e
[docs]    @classmethod
e,
            )
            raise e
[docs]    @classmethod
    def from_texts(
        cls,
        texts: List[str],
        embedding: Embeddings,
        metadatas: Optional[List[dict]] = None,
        config: Optional[AlibabaCloudOpenSearchSettings] = None,
        **kwargs: Any,
    ) -> "AlibabaCloudOpenSearch":
        """Create alibaba cloud opensearch vector store instance.
        Args:
            texts: The text segments to be inserted into the vector storage,
             should not be empty.
            embedding: Embedding function, Embedding function.
            config: Alibaba OpenSearch instance configuration.
            metadatas: Metadata information.
        Returns:
            AlibabaCloudOpenSearch: Alibaba cloud opensearch vector store instance.
        """
        if texts is None or len(texts) == 0:
            raise Exception("the inserted text segments, should not be empty.")
        if embedding is None:
            raise Exception("the embeddings should not be empty.")
        if config is None:
            raise Exception("config should not be none.")
        ctx = cls(embedding, config, **kwargs)
        ctx.add_texts(texts=texts, metadatas=metadatas)
        return ctx
[docs]    @classmethod
    def from_documents(
        cls,
        documents: List[Document],
        embedding: Embeddings,
        config: Optional[AlibabaCloudOpenSearchSettings] = None,
        **kwargs: Any,
    ) -> "AlibabaCloudOpenSearch":
        """Create alibaba cloud opensearch vector store instance.
        Args:
            documents: Documents to be inserted into the vector storage,
             should not be empty.
documents: Documents to be inserted into the vector storage,
             should not be empty.
            embedding: Embedding function, Embedding function.
            config: Alibaba OpenSearch instance configuration.
            ids: Specify the ID for the inserted document. If left empty, the ID will be
             automatically generated based on the text content.
        Returns:
            AlibabaCloudOpenSearch: Alibaba cloud opensearch vector store instance.
        """
        if documents is None or len(documents) == 0:
            raise Exception("the inserted documents, should not be empty.")
        if embedding is None:
            raise Exception("the embeddings should not be empty.")
        if config is None:
            raise Exception("config can't be none")
        texts = [d.page_content for d in documents]
        metadatas = [d.metadata for d in documents]
        return cls.from_texts(
            texts=texts,
            embedding=embedding,
            metadatas=metadatas,
            config=config,
            **kwargs,
        )
Source code for langchain_community.vectorstores.baiducloud_vector_search
import logging
import uuid
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Dict,
    Iterable,
    List,
    Optional,
    Tuple,
    Union,
)
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.vectorstores import VectorStore
if TYPE_CHECKING:
    from elasticsearch import Elasticsearch
logger = logging.getLogger(__name__)
[docs]class BESVectorStore(VectorStore):
    """`Baidu Elasticsearch` vector store.
    Example:
        .. code-block:: python
            from langchain_community.vectorstores import BESVectorStore
            from langchain_community.embeddings.openai import OpenAIEmbeddings
            embeddings = OpenAIEmbeddings()
            vectorstore = BESVectorStore(
                embedding=OpenAIEmbeddings(),
                index_name="langchain-demo",
                bes_url="http://localhost:9200"
            )
    Args:
        index_name: Name of the Elasticsearch index to create.
        bes_url: URL of the Baidu Elasticsearch instance to connect to.
        user: Username to use when connecting to Elasticsearch.
        password: Password to use when connecting to Elasticsearch.
        More information can be obtained from:
        https://cloud.baidu.com/doc/BES/s/8llyn0hh4
    """
[docs]    def __init__(
        self,
        index_name: str,
        bes_url: str,
        user: Optional[str] = None,
        password: Optional[str] = None,
        embedding: Optional[Embeddings] = None,
        **kwargs: Optional[dict],
    ) -> None:
        self.embedding = embedding
) -> None:
        self.embedding = embedding
        self.index_name = index_name
        self.query_field = kwargs.get("query_field", "text")
        self.vector_query_field = kwargs.get("vector_query_field", "vector")
        self.space_type = kwargs.get("space_type", "cosine")
        self.index_type = kwargs.get("index_type", "linear")
        self.index_params = kwargs.get("index_params") or {}
        if bes_url is not None:
            self.client = BESVectorStore.bes_client(
                bes_url=bes_url, username=user, password=password
            )
        else:
            raise ValueError("""Please specified a bes connection url.""")
    @property
    def embeddings(self) -> Optional[Embeddings]:
        return self.embedding
[docs]    @staticmethod
    def bes_client(
        *,
        bes_url: Optional[str] = None,
        username: Optional[str] = None,
        password: Optional[str] = None,
    ) -> "Elasticsearch":
        try:
            import elasticsearch
        except ImportError:
            raise ImportError(
                "Could not import elasticsearch python package. "
                "Please install it with `pip install elasticsearch`."
            )
        connection_params: Dict[str, Any] = {}
        connection_params["hosts"] = [bes_url]
        if username and password:
            connection_params["basic_auth"] = (username, password)
        es_client = elasticsearch.Elasticsearch(**connection_params)
        try:
            es_client.info()
        except Exception as e:
            logger.error(f"Error connecting to Elasticsearch: {e}")
            raise e
        return es_client
    def _create_index_if_not_exists(self, dims_length: Optional[int] = None) -> None:
"""Create the index if it doesn't already exist.
        Args:
            dims_length: Length of the embedding vectors.
        """
        if self.client.indices.exists(index=self.index_name):
            logger.info(f"Index {self.index_name} already exists. Skipping creation.")
        else:
            if dims_length is None:
                raise ValueError(
                    "Cannot create index without specifying dims_length "
                    + "when the index doesn't already exist. "
                )
            indexMapping = self._index_mapping(dims_length=dims_length)
            logger.debug(
                f"Creating index {self.index_name} with mappings {indexMapping}"
            )
            self.client.indices.create(
                index=self.index_name,
                body={
                    "settings": {"index": {"knn": True}},
                    "mappings": {"properties": indexMapping},
                },
            )
    def _index_mapping(self, dims_length: Union[int, None]) -> Dict:
        """
        Executes when the index is created.
        Args:
            dims_length: Numeric length of the embedding vectors,
                        or None if not using vector-based query.
            index_params: The extra pamameters for creating index.
        Returns:
            Dict: The Elasticsearch settings and mappings for the strategy.
        """
        if "linear" == self.index_type:
            return {
                self.vector_query_field: {
                    "type": "bpack_vector",
                    "dims": dims_length,
                    "build_index": self.index_params.get("build_index", False),
                }
            }
        elif "hnsw" == self.index_type:
            return {
                self.vector_query_field: {
                    "type": "bpack_vector",
                    "dims": dims_length,
"type": "bpack_vector",
                    "dims": dims_length,
                    "index_type": "hnsw",
                    "space_type": self.space_type,
                    "parameters": {
                        "ef_construction": self.index_params.get(
                            "hnsw_ef_construction", 200
                        ),
                        "m": self.index_params.get("hnsw_m", 4),
                    },
                }
            }
        else:
            return {
                self.vector_query_field: {
                    "type": "bpack_vector",
                    "model_id": self.index_params.get("model_id", ""),
                }
            }
[docs]    def delete(
        self,
        ids: Optional[List[str]] = None,
        **kwargs: Any,
    ) -> Optional[bool]:
        """Delete documents from the index.
        Args:
            ids: List of ids of documents to delete
        """
        try:
            from elasticsearch.helpers import BulkIndexError, bulk
        except ImportError:
            raise ImportError(
                "Could not import elasticsearch python package. "
                "Please install it with `pip install elasticsearch`."
            )
        body = []
        if ids is None:
            raise ValueError("ids must be provided.")
        for _id in ids:
            body.append({"_op_type": "delete", "_index": self.index_name, "_id": _id})
        if len(body) > 0:
            try:
                bulk(
                    self.client,
                    body,
                    refresh=kwargs.get("refresh_indices", True),
                    ignore_status=404,
                )
                logger.debug(f"Deleted {len(body)} texts from index")
                return True
            except BulkIndexError as e:
return True
            except BulkIndexError as e:
                logger.error(f"Error deleting texts: {e}")
                raise e
        else:
            logger.info("No documents to delete")
            return False
    def _query_body(
        self,
        query_vector: Union[List[float], None],
        filter: Optional[dict] = None,
        search_params: Dict = {},
    ) -> Dict:
        query_vector_body = {"vector": query_vector, "k": search_params.get("k", 2)}
        if filter is not None and len(filter) != 0:
            query_vector_body["filter"] = filter
        if "linear" == self.index_type:
            query_vector_body["linear"] = True
        else:
            query_vector_body["ef"] = search_params.get("ef", 10)
        return {
            "size": search_params.get("size", 4),
            "query": {"knn": {self.vector_query_field: query_vector_body}},
        }
    def _search(
        self,
        query: Optional[str] = None,
        query_vector: Union[List[float], None] = None,
        filter: Optional[dict] = None,
        custom_query: Optional[Callable[[Dict, Union[str, None]], Dict]] = None,
        search_params: Dict = {},
    ) -> List[Tuple[Document, float]]:
        """Return searched documents result from BES
        Args:
            query: Text to look up documents similar to.
            query_vector: Embedding to look up documents similar to.
            filter: Array of Baidu ElasticSearch filter clauses to apply to the query.
            custom_query: Function to modify the query body before it is sent to BES.
        Returns: