viruPVS/langchain-docs huggingface dataset:
>>> filter = RedisNum("age") > 18
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.GT)  # type: ignore
        return RedisFilterExpression(str(self))
    def __lt__(self, other: Union[int, float]) -> "RedisFilterExpression":
        """Create a Numeric less than filter expression.
        Args:
            other (Union[int, float]): The value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisNum
            >>> filter = RedisNum("age") < 18
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.LT)  # type: ignore
        return RedisFilterExpression(str(self))
    def __ge__(self, other: Union[int, float]) -> "RedisFilterExpression":
        """Create a Numeric greater than or equal to filter expression.
        Args:
            other (Union[int, float]): The value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisNum
            >>> filter = RedisNum("age") >= 18
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.GE)  # type: ignore
        return RedisFilterExpression(str(self))
    def __le__(self, other: Union[int, float]) -> "RedisFilterExpression":
        """Create a Numeric less than or equal to filter expression.
        Args:
            other (Union[int, float]): The value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisNum
            >>> filter = RedisNum("age") <= 18
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.LE)  # type: ignore
return RedisFilterExpression(str(self))
[docs]class RedisText(RedisFilterField):
    """A RedisFilterField representing a text field in a Redis index."""
    OPERATORS: Dict[RedisFilterOperator, str] = {
        RedisFilterOperator.EQ: "==",
        RedisFilterOperator.NE: "!=",
        RedisFilterOperator.LIKE: "%",
    }
    OPERATOR_MAP: Dict[RedisFilterOperator, str] = {
        RedisFilterOperator.EQ: '@%s:("%s")',
        RedisFilterOperator.NE: '(-@%s:"%s")',
        RedisFilterOperator.LIKE: "@%s:(%s)",
    }
    SUPPORTED_VAL_TYPES = (str, type(None))
    @check_operator_misuse
    def __eq__(self, other: str) -> "RedisFilterExpression":
        """Create a RedisText equality (exact match) filter expression.
        Args:
            other (str): The text value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisText
            >>> filter = RedisText("job") == "engineer"
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.EQ)  # type: ignore
        return RedisFilterExpression(str(self))
    @check_operator_misuse
    def __ne__(self, other: str) -> "RedisFilterExpression":
        """Create a RedisText inequality filter expression.
        Args:
            other (str): The text value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisText
            >>> filter = RedisText("job") != "engineer"
        """
>>> filter = RedisText("job") != "engineer"
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.NE)  # type: ignore
        return RedisFilterExpression(str(self))
    def __mod__(self, other: str) -> "RedisFilterExpression":
        """Create a RedisText "LIKE" filter expression.
        Args:
            other (str): The text value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisText
            >>> filter = RedisText("job") % "engine*"         # suffix wild card match
            >>> filter = RedisText("job") % "%%engine%%"      # fuzzy match w/ LD
            >>> filter = RedisText("job") % "engineer|doctor" # contains either term
            >>> filter = RedisText("job") % "engineer doctor" # contains both terms
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.LIKE)  # type: ignore
        return RedisFilterExpression(str(self))
    def __str__(self) -> str:
        """Return the query syntax for a RedisText filter expression."""
        if not self._value:
            return "*"
        return self.OPERATOR_MAP[self._operator] % (
            self._field,
            self._value,
        )
[docs]class RedisFilterExpression:
    """A logical expression of RedisFilterFields.
    RedisFilterExpressions can be combined using the & and | operators to create
    complex logical expressions that evaluate to the Redis Query language.
    This presents an interface by which users can create complex queries
    without having to know the Redis Query language.
    Filter expressions are not initialized directly. Instead they are built
Filter expressions are not initialized directly. Instead they are built
    by combining RedisFilterFields using the & and | operators.
    Examples:
        >>> from langchain_community.vectorstores.redis import RedisTag, RedisNum
        >>> brand_is_nike = RedisTag("brand") == "nike"
        >>> price_is_under_100 = RedisNum("price") < 100
        >>> filter = brand_is_nike & price_is_under_100
        >>> print(str(filter))
        (@brand:{nike} @price:[-inf (100)])
    """
[docs]    def __init__(
        self,
        _filter: Optional[str] = None,
        operator: Optional[RedisFilterOperator] = None,
        left: Optional["RedisFilterExpression"] = None,
        right: Optional["RedisFilterExpression"] = None,
    ):
        self._filter = _filter
        self._operator = operator
        self._left = left
        self._right = right
    def __and__(self, other: "RedisFilterExpression") -> "RedisFilterExpression":
        return RedisFilterExpression(
            operator=RedisFilterOperator.AND, left=self, right=other
        )
    def __or__(self, other: "RedisFilterExpression") -> "RedisFilterExpression":
        return RedisFilterExpression(
            operator=RedisFilterOperator.OR, left=self, right=other
        )
[docs]    @staticmethod
    def format_expression(
        left: "RedisFilterExpression", right: "RedisFilterExpression", operator_str: str
    ) -> str:
        _left, _right = str(left), str(right)
        if _left == _right == "*":
            return _left
        if _left == "*" != _right:
            return _right
if _left == "*" != _right:
            return _right
        if _right == "*" != _left:
            return _left
        return f"({_left}{operator_str}{_right})"
    def __str__(self) -> str:
        # top level check that allows recursive calls to __str__
        if not self._filter and not self._operator:
            raise ValueError("Improperly initialized RedisFilterExpression")
        # if there's an operator, combine expressions accordingly
        if self._operator:
            if not isinstance(self._left, RedisFilterExpression) or not isinstance(
                self._right, RedisFilterExpression
            ):
                raise TypeError(
                    "Improper combination of filters."
                    "Both left and right should be type FilterExpression"
                )
            operator_str = " | " if self._operator == RedisFilterOperator.OR else " "
            return self.format_expression(self._left, self._right, operator_str)
        # check that base case, the filter is set
        if not self._filter:
            raise ValueError("Improperly initialized RedisFilterExpression")
        return self._filter
Source code for langchain_community.vectorstores.redis.schema
from __future__ import annotations
import os
from enum import Enum
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
import numpy as np
import yaml
from langchain_core.pydantic_v1 import BaseModel, Field, validator
from typing_extensions import TYPE_CHECKING, Literal
from langchain_community.vectorstores.redis.constants import REDIS_VECTOR_DTYPE_MAP
if TYPE_CHECKING:
    from redis.commands.search.field import (  # type: ignore
        NumericField,
        TagField,
        TextField,
        VectorField,
    )
[docs]class RedisDistanceMetric(str, Enum):
    """Distance metrics for Redis vector fields."""
    l2 = "L2"
    cosine = "COSINE"
    ip = "IP"
[docs]class RedisField(BaseModel):
    """Base class for Redis fields."""
    name: str = Field(...)
[docs]class TextFieldSchema(RedisField):
    """Schema for text fields in Redis."""
    weight: float = 1
    no_stem: bool = False
    phonetic_matcher: Optional[str] = None
    withsuffixtrie: bool = False
    no_index: bool = False
    sortable: Optional[bool] = False
[docs]    def as_field(self) -> TextField:
        from redis.commands.search.field import TextField  # type: ignore
        return TextField(
            self.name,
            weight=self.weight,
            no_stem=self.no_stem,
            phonetic_matcher=self.phonetic_matcher,  # type: ignore
            sortable=self.sortable,
            no_index=self.no_index,
        )
[docs]class TagFieldSchema(RedisField):
    """Schema for tag fields in Redis."""
"""Schema for tag fields in Redis."""
    separator: str = ","
    case_sensitive: bool = False
    no_index: bool = False
    sortable: Optional[bool] = False
[docs]    def as_field(self) -> TagField:
        from redis.commands.search.field import TagField  # type: ignore
        return TagField(
            self.name,
            separator=self.separator,
            case_sensitive=self.case_sensitive,
            sortable=self.sortable,
            no_index=self.no_index,
        )
[docs]class NumericFieldSchema(RedisField):
    """Schema for numeric fields in Redis."""
    no_index: bool = False
    sortable: Optional[bool] = False
[docs]    def as_field(self) -> NumericField:
        from redis.commands.search.field import NumericField  # type: ignore
        return NumericField(self.name, sortable=self.sortable, no_index=self.no_index)
[docs]class RedisVectorField(RedisField):
    """Base class for Redis vector fields."""
    dims: int = Field(...)
    algorithm: object = Field(...)
    datatype: str = Field(default="FLOAT32")
    distance_metric: RedisDistanceMetric = Field(default="COSINE")
    initial_cap: Optional[int] = None
    @validator("algorithm", "datatype", "distance_metric", pre=True, each_item=True)
    def uppercase_strings(cls, v: str) -> str:
        return v.upper()
    @validator("datatype", pre=True)
    def uppercase_and_check_dtype(cls, v: str) -> str:
        if v.upper() not in REDIS_VECTOR_DTYPE_MAP:
            raise ValueError(
                f"datatype must be one of {REDIS_VECTOR_DTYPE_MAP.keys()}. Got {v}"
            )
        return v.upper()
)
        return v.upper()
    def _fields(self) -> Dict[str, Any]:
        field_data = {
            "TYPE": self.datatype,
            "DIM": self.dims,
            "DISTANCE_METRIC": self.distance_metric,
        }
        if self.initial_cap is not None:  # Only include it if it's set
            field_data["INITIAL_CAP"] = self.initial_cap
        return field_data
[docs]class FlatVectorField(RedisVectorField):
    """Schema for flat vector fields in Redis."""
    algorithm: Literal["FLAT"] = "FLAT"
    block_size: Optional[int] = None
[docs]    def as_field(self) -> VectorField:
        from redis.commands.search.field import VectorField  # type: ignore
        field_data = super()._fields()
        if self.block_size is not None:
            field_data["BLOCK_SIZE"] = self.block_size
        return VectorField(self.name, self.algorithm, field_data)
[docs]class HNSWVectorField(RedisVectorField):
    """Schema for HNSW vector fields in Redis."""
    algorithm: Literal["HNSW"] = "HNSW"
    m: int = Field(default=16)
    ef_construction: int = Field(default=200)
    ef_runtime: int = Field(default=10)
    epsilon: float = Field(default=0.01)
[docs]    def as_field(self) -> VectorField:
        from redis.commands.search.field import VectorField  # type: ignore
        field_data = super()._fields()
        field_data.update(
            {
                "M": self.m,
                "EF_CONSTRUCTION": self.ef_construction,
                "EF_RUNTIME": self.ef_runtime,
                "EPSILON": self.epsilon,
"EF_RUNTIME": self.ef_runtime,
                "EPSILON": self.epsilon,
            }
        )
        return VectorField(self.name, self.algorithm, field_data)
[docs]class RedisModel(BaseModel):
    """Schema for Redis index."""
    # always have a content field for text
    text: List[TextFieldSchema] = [TextFieldSchema(name="content")]
    tag: Optional[List[TagFieldSchema]] = None
    numeric: Optional[List[NumericFieldSchema]] = None
    extra: Optional[List[RedisField]] = None
    # filled by default_vector_schema
    vector: Optional[List[Union[FlatVectorField, HNSWVectorField]]] = None
    content_key: str = "content"
    content_vector_key: str = "content_vector"
[docs]    def add_content_field(self) -> None:
        if self.text is None:
            self.text = []
        for field in self.text:
            if field.name == self.content_key:
                return
        self.text.append(TextFieldSchema(name=self.content_key))
[docs]    def add_vector_field(self, vector_field: Dict[str, Any]) -> None:
        # catch case where user inputted no vector field spec
        # in the index schema
        if self.vector is None:
            self.vector = []
        # ignore types as pydantic is handling type validation and conversion
        if vector_field["algorithm"] == "FLAT":
            self.vector.append(FlatVectorField(**vector_field))  # type: ignore
        elif vector_field["algorithm"] == "HNSW":
            self.vector.append(HNSWVectorField(**vector_field))  # type: ignore
        else:
            raise ValueError(
                f"algorithm must be either FLAT or HNSW. Got "
f"algorithm must be either FLAT or HNSW. Got "
                f"{vector_field['algorithm']}"
            )
[docs]    def as_dict(self) -> Dict[str, List[Any]]:
        schemas: Dict[str, List[Any]] = {"text": [], "tag": [], "numeric": []}
        # iter over all class attributes
        for attr, attr_value in self.__dict__.items():
            # only non-empty lists
            if isinstance(attr_value, list) and len(attr_value) > 0:
                field_values: List[Dict[str, Any]] = []
                # iterate over all fields in each category (tag, text, etc)
                for val in attr_value:
                    value: Dict[str, Any] = {}
                    # iterate over values within each field to extract
                    # settings for that field (i.e. name, weight, etc)
                    for field, field_value in val.__dict__.items():
                        # make enums into strings
                        if isinstance(field_value, Enum):
                            value[field] = field_value.value
                        # don't write null values
                        elif field_value is not None:
                            value[field] = field_value
                    field_values.append(value)
                schemas[attr] = field_values
        schema: Dict[str, List[Any]] = {}
        # only write non-empty lists from defaults
        for k, v in schemas.items():
            if len(v) > 0:
                schema[k] = v
        return schema
    @property
    def content_vector(self) -> Union[FlatVectorField, HNSWVectorField]:
        if not self.vector:
            raise ValueError("No vector fields found")
        for field in self.vector:
            if field.name == self.content_vector_key:
                return field
if field.name == self.content_vector_key:
                return field
        raise ValueError("No content_vector field found")
    @property
    def vector_dtype(self) -> np.dtype:
        # should only ever be called after pydantic has validated the schema
        return REDIS_VECTOR_DTYPE_MAP[self.content_vector.datatype]
    @property
    def is_empty(self) -> bool:
        return all(
            field is None for field in [self.tag, self.text, self.numeric, self.vector]
        )
[docs]    def get_fields(self) -> List["RedisField"]:
        redis_fields: List["RedisField"] = []
        if self.is_empty:
            return redis_fields
        for field_name in self.__fields__.keys():
            if field_name not in ["content_key", "content_vector_key", "extra"]:
                field_group = getattr(self, field_name)
                if field_group is not None:
                    for field in field_group:
                        redis_fields.append(field.as_field())
        return redis_fields
    @property
    def metadata_keys(self) -> List[str]:
        keys: List[str] = []
        if self.is_empty:
            return keys
        for field_name in self.__fields__.keys():
            field_group = getattr(self, field_name)
            if field_group is not None:
                for field in field_group:
                    # check if it's a metadata field. exclude vector and content key
                    if not isinstance(field, str) and field.name not in [
                        self.content_key,
                        self.content_vector_key,
                    ]:
                        keys.append(field.name)
        return keys
[docs]def read_schema(
    index_schema: Optional[Union[Dict[str, List[Any]], str, os.PathLike]],
) -> Dict[str, Any]:
) -> Dict[str, Any]:
    """Reads in the index schema from a dict or yaml file.
    Check if it is a dict and return RedisModel otherwise, check if it's a path and
    read in the file assuming it's a yaml file and return a RedisModel
    """
    if isinstance(index_schema, dict):
        return index_schema
    elif isinstance(index_schema, Path):
        with open(index_schema, "rb") as f:
            return yaml.safe_load(f)
    elif isinstance(index_schema, str):
        if Path(index_schema).resolve().is_file():
            with open(index_schema, "rb") as f:
                return yaml.safe_load(f)
        else:
            raise FileNotFoundError(f"index_schema file {index_schema} does not exist")
    else:
        raise TypeError(
            f"index_schema must be a dict, or path to a yaml file "
            f"Got {type(index_schema)}"
        )
Source code for langchain_community.vectorstores.docarray.base
from abc import ABC
from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Type
import numpy as np
from langchain_core.documents import Document
from langchain_core.embeddings import Embeddings
from langchain_core.pydantic_v1 import Field
from langchain_core.vectorstores import VectorStore
from langchain_community.vectorstores.utils import maximal_marginal_relevance
if TYPE_CHECKING:
    from docarray import BaseDoc
    from docarray.index.abstract import BaseDocIndex
def _check_docarray_import() -> None:
    try:
        import docarray
        da_version = docarray.__version__.split(".")
        if int(da_version[0]) == 0 and int(da_version[1]) <= 31:
            raise ImportError(
                f"To use the DocArrayHnswSearch VectorStore the docarray "
                f"version >=0.32.0 is expected, received: {docarray.__version__}."
                f"To upgrade, please run: `pip install -U docarray`."
            )
    except ImportError:
        raise ImportError(
            "Could not import docarray python package. "
            'Please install it with `pip install "langchain[docarray]"`.'
        )
[docs]class DocArrayIndex(VectorStore, ABC):
    """Base class for `DocArray` based vector stores."""
[docs]    def __init__(
        self,
        doc_index: "BaseDocIndex",
        embedding: Embeddings,
    ):
        """Initialize a vector store from DocArray's DocIndex."""
        self.doc_index = doc_index
        self.embedding = embedding
    @staticmethod
    def _get_doc_cls(**embeddings_params: Any) -> Type["BaseDoc"]:
def _get_doc_cls(**embeddings_params: Any) -> Type["BaseDoc"]:
        """Get docarray Document class describing the schema of DocIndex."""
        from docarray import BaseDoc
        from docarray.typing import NdArray
        class DocArrayDoc(BaseDoc):
            text: Optional[str]
            embedding: Optional[NdArray] = Field(**embeddings_params)
            metadata: Optional[dict]
        return DocArrayDoc
    @property
    def doc_cls(self) -> Type["BaseDoc"]:
        if self.doc_index._schema is None:
            raise ValueError("doc_index expected to have non-null _schema attribute.")
        return self.doc_index._schema
[docs]    def add_texts(
        self,
        texts: Iterable[str],
        metadatas: Optional[List[dict]] = None,
        **kwargs: Any,
    ) -> List[str]:
        """Embed texts and add to the vector store.
        Args:
            texts: Iterable of strings to add to the vectorstore.
            metadatas: Optional list of metadatas associated with the texts.
        Returns:
            List of ids from adding the texts into the vectorstore.
        """
        ids: List[str] = []
        embeddings = self.embedding.embed_documents(list(texts))
        for i, (t, e) in enumerate(zip(texts, embeddings)):
            m = metadatas[i] if metadatas else {}
            doc = self.doc_cls(text=t, embedding=e, metadata=m)
            self.doc_index.index([doc])
            ids.append(str(doc.id))
        return ids
[docs]    def similarity_search_with_score(
        self, query: str, k: int = 4, **kwargs: Any
self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Tuple[Document, float]]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of documents most similar to the query text and
            cosine distance in float for each.
            Lower score represents more similarity.
        """
        query_embedding = self.embedding.embed_query(query)
        query_doc = self.doc_cls(embedding=query_embedding)  # type: ignore
        docs, scores = self.doc_index.find(query_doc, search_field="embedding", limit=k)
        result = [
            (Document(page_content=doc.text, metadata=doc.metadata), score)
            for doc, score in zip(docs, scores)
        ]
        return result
[docs]    def similarity_search(
        self, query: str, k: int = 4, **kwargs: Any
    ) -> List[Document]:
        """Return docs most similar to query.
        Args:
            query: Text to look up documents similar to.
            k: Number of Documents to return. Defaults to 4.
        Returns:
            List of Documents most similar to the query.
        """
        results = self.similarity_search_with_score(query, k=k, **kwargs)
        return [doc for doc, _ in results]
    def _similarity_search_with_relevance_scores(
        self,
        query: str,
        k: int = 4,
        **kwargs: Any,
    ) -> List[Tuple[Document, float]]:
        """Return docs and relevance scores, normalized on a scale from 0 to 1.