viruPVS/langchain-docs huggingface dataset:
+ f"{self._schema.metadata_keys}"
                )
                meta[key] = None
        return meta
    def _prepare_query(
        self,
        query_embedding: List[float],
        k: int = 4,
        filter: Optional[RedisFilterExpression] = None,
        distance_threshold: Optional[float] = None,
        with_metadata: bool = True,
        with_distance: bool = False,
    ) -> Tuple["Query", Dict[str, Any]]:
        # Creates Redis query
        params_dict: Dict[str, Union[str, bytes, float]] = {
            "vector": _array_to_buffer(query_embedding, self._schema.vector_dtype),
        }
        # prepare return fields including score
        return_fields = [self._schema.content_key]
        if with_distance:
            return_fields.append("distance")
        if with_metadata:
            return_fields.extend(self._schema.metadata_keys)
        if distance_threshold:
            params_dict["distance_threshold"] = distance_threshold
            return (
                self._prepare_range_query(
                    k, filter=filter, return_fields=return_fields
                ),
                params_dict,
            )
        return (
            self._prepare_vector_query(k, filter=filter, return_fields=return_fields),
            params_dict,
        )
    def _prepare_range_query(
        self,
        k: int,
        filter: Optional[RedisFilterExpression] = None,
        return_fields: Optional[List[str]] = None,
    ) -> "Query":
        try:
            from redis.commands.search.query import Query
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
"Please install it with `pip install redis`."
            ) from e
        return_fields = return_fields or []
        vector_key = self._schema.content_vector_key
        base_query = f"@{vector_key}:[VECTOR_RANGE $distance_threshold $vector]"
        if filter:
            base_query = "(" + base_query + " " + str(filter) + ")"
        query_string = base_query + "=>{$yield_distance_as: distance}"
        return (
            Query(query_string)
            .return_fields(*return_fields)
            .sort_by("distance")
            .paging(0, k)
            .dialect(2)
        )
    def _prepare_vector_query(
        self,
        k: int,
        filter: Optional[RedisFilterExpression] = None,
        return_fields: Optional[List[str]] = None,
    ) -> "Query":
        """Prepare query for vector search.
        Args:
            k: Number of results to return.
            filter: Optional metadata filter.
        Returns:
            query: Query object.
        """
        try:
            from redis.commands.search.query import Query
        except ImportError as e:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            ) from e
        return_fields = return_fields or []
        query_prefix = "*"
        if filter:
            query_prefix = f"{str(filter)}"
        vector_key = self._schema.content_vector_key
        base_query = f"({query_prefix})=>[KNN {k} @{vector_key} $vector AS distance]"
        query = (
            Query(base_query)
            .return_fields(*return_fields)
            .sort_by("distance")
            .paging(0, k)
.sort_by("distance")
            .paging(0, k)
            .dialect(2)
        )
        return query
    def _get_schema_with_defaults(
        self,
        index_schema: Optional[Union[Dict[str, str], str, os.PathLike]] = None,
        vector_schema: Optional[Dict[str, Union[str, int]]] = None,
    ) -> "RedisModel":
        # should only be called after init of Redis (so Import handled)
        from langchain_community.vectorstores.redis.schema import (
            RedisModel,
            read_schema,
        )
        schema = RedisModel()
        # read in schema (yaml file or dict) and
        # pass to the Pydantic validators
        if index_schema:
            schema_values = read_schema(index_schema)  # type: ignore
            schema = RedisModel(**schema_values)
            # ensure user did not exclude the content field
            # no modifications if content field found
            schema.add_content_field()
        # if no content_vector field, add vector field to schema
        # this makes adding a vector field to the schema optional when
        # the user just wants additional metadata
        try:
            # see if user overrode the content vector
            schema.content_vector
            # if user overrode the content vector, check if they
            # also passed vector schema. This won't be used since
            # the index schema overrode the content vector
            if vector_schema:
                logger.warning(
                    "`vector_schema` is ignored since content_vector is "
                    + "overridden in `index_schema`."
                )
        # user did not override content vector
        except ValueError:
            # set default vector schema and update with user provided schema
            # if the user provided any
# if the user provided any
            vector_field = self.DEFAULT_VECTOR_SCHEMA.copy()
            if vector_schema:
                vector_field.update(vector_schema)
            # add the vector field either way
            schema.add_vector_field(vector_field)
        return schema
    def _create_index_if_not_exist(self, dim: int = 1536) -> None:
        try:
            from redis.commands.search.indexDefinition import (  # type: ignore
                IndexDefinition,
                IndexType,
            )
        except ImportError:
            raise ImportError(
                "Could not import redis python package. "
                "Please install it with `pip install redis`."
            )
        # Set vector dimension
        # can't obtain beforehand because we don't
        # know which embedding model is being used.
        self._schema.content_vector.dims = dim
        # Check if index exists
        if not check_index_exists(self.client, self.index_name):
            # Create Redis Index
            self.client.ft(self.index_name).create_index(
                fields=self._schema.get_fields(),
                definition=IndexDefinition(
                    prefix=[self.key_prefix], index_type=IndexType.HASH
                ),
            )
    def _calculate_fp_distance(self, distance: str) -> float:
        """Calculate the distance based on the vector datatype
        Two datatypes supported:
        - FLOAT32
        - FLOAT64
        if it's FLOAT32, we need to round the distance to 4 decimal places
        otherwise, round to 7 decimal places.
        """
        if self._schema.content_vector.datatype == "FLOAT32":
            return round(float(distance), 4)
        return round(float(distance), 7)
    def _check_deprecated_kwargs(self, kwargs: Mapping[str, Any]) -> None:
def _check_deprecated_kwargs(self, kwargs: Mapping[str, Any]) -> None:
        """Check for deprecated kwargs."""
        deprecated_kwargs = {
            "redis_host": "redis_url",
            "redis_port": "redis_url",
            "redis_password": "redis_url",
            "content_key": "index_schema",
            "vector_key": "vector_schema",
            "distance_metric": "vector_schema",
        }
        for key, value in kwargs.items():
            if key in deprecated_kwargs:
                raise ValueError(
                    f"Keyword argument '{key}' is deprecated. "
                    f"Please use '{deprecated_kwargs[key]}' instead."
                )
    def _select_relevance_score_fn(self) -> Callable[[float], float]:
        if self.relevance_score_fn:
            return self.relevance_score_fn
        metric_map = {
            "COSINE": self._cosine_relevance_score_fn,
            "IP": self._max_inner_product_relevance_score_fn,
            "L2": self._euclidean_relevance_score_fn,
        }
        try:
            return metric_map[self._schema.content_vector.distance_metric]
        except KeyError:
            return _default_relevance_score
def _generate_field_schema(data: Dict[str, Any]) -> Dict[str, Any]:
    """
    Generate a schema for the search index in Redis based on the input metadata.
    Given a dictionary of metadata, this function categorizes each metadata
        field into one of the three categories:
    - text: The field contains textual data.
    - numeric: The field contains numeric data (either integer or float).
    - tag: The field contains list of tags (strings).
    Args
        data (Dict[str, Any]): A dictionary where keys are metadata field names
Args
        data (Dict[str, Any]): A dictionary where keys are metadata field names
            and values are the metadata values.
    Returns:
        Dict[str, Any]: A dictionary with three keys "text", "numeric", and "tag".
            Each key maps to a list of fields that belong to that category.
    Raises:
        ValueError: If a metadata field cannot be categorized into any of
            the three known types.
    """
    result: Dict[str, Any] = {
        "text": [],
        "numeric": [],
        "tag": [],
    }
    for key, value in data.items():
        # Numeric fields
        try:
            int(value)
            result["numeric"].append({"name": key})
            continue
        except (ValueError, TypeError):
            pass
        # None values are not indexed as of now
        if value is None:
            continue
        # if it's a list of strings, we assume it's a tag
        if isinstance(value, (list, tuple)):
            if not value or isinstance(value[0], str):
                result["tag"].append({"name": key})
            else:
                name = type(value[0]).__name__
                raise ValueError(
                    f"List/tuple values should contain strings: '{key}': {name}"
                )
            continue
        # Check if value is string before processing further
        if isinstance(value, str):
            result["text"].append({"name": key})
            continue
        # Unable to classify the field value
        name = type(value).__name__
        raise ValueError(
            "Could not generate Redis index field type mapping "
            + f"for metadata: '{key}': {name}"
        )
    return result
)
    return result
def _prepare_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
    """
    Prepare metadata for indexing in Redis by sanitizing its values.
    - String, integer, and float values remain unchanged.
    - None or empty values are replaced with empty strings.
    - Lists/tuples of strings are joined into a single string with a comma separator.
    Args:
        metadata (Dict[str, Any]): A dictionary where keys are metadata
            field names and values are the metadata values.
    Returns:
        Dict[str, Any]: A sanitized dictionary ready for indexing in Redis.
    Raises:
        ValueError: If any metadata value is not one of the known
            types (string, int, float, or list of strings).
    """
    def raise_error(key: str, value: Any) -> None:
        raise ValueError(
            f"Metadata value for key '{key}' must be a string, int, "
            + f"float, or list of strings. Got {type(value).__name__}"
        )
    clean_meta: Dict[str, Union[str, float, int]] = {}
    for key, value in metadata.items():
        if value is None:
            clean_meta[key] = ""
            continue
        # No transformation needed
        if isinstance(value, (str, int, float)):
            clean_meta[key] = value
        # if it's a list/tuple of strings, we join it
        elif isinstance(value, (list, tuple)):
            if not value or isinstance(value[0], str):
                clean_meta[key] = REDIS_TAG_SEPARATOR.join(value)
            else:
                raise_error(key, value)
        else:
            raise_error(key, value)
    return clean_meta
else:
            raise_error(key, value)
    return clean_meta
[docs]class RedisVectorStoreRetriever(VectorStoreRetriever):
    """Retriever for Redis VectorStore."""
    vectorstore: Redis
    """Redis VectorStore."""
    search_type: str = "similarity"
    """Type of search to perform. Can be either
    'similarity',
    'similarity_distance_threshold',
    'similarity_score_threshold'
    """
    search_kwargs: Dict[str, Any] = {
        "k": 4,
        "score_threshold": 0.9,
        # set to None to avoid distance used in score_threshold search
        "distance_threshold": None,
    }
    """Default search kwargs."""
    allowed_search_types = [
        "similarity",
        "similarity_distance_threshold",
        "similarity_score_threshold",
        "mmr",
    ]
    """Allowed search types."""
    class Config:
        """Configuration for this pydantic object."""
        arbitrary_types_allowed = True
    def _get_relevant_documents(
        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
    ) -> List[Document]:
        if self.search_type == "similarity":
            docs = self.vectorstore.similarity_search(query, **self.search_kwargs)
        elif self.search_type == "similarity_distance_threshold":
            if self.search_kwargs["distance_threshold"] is None:
                raise ValueError(
                    "distance_threshold must be provided for "
                    + "similarity_distance_threshold retriever"
                )
            docs = self.vectorstore.similarity_search(query, **self.search_kwargs)
        elif self.search_type == "similarity_score_threshold":
            docs_and_similarities = (
docs_and_similarities = (
                self.vectorstore.similarity_search_with_relevance_scores(
                    query, **self.search_kwargs
                )
            )
            docs = [doc for doc, _ in docs_and_similarities]
        elif self.search_type == "mmr":
            docs = self.vectorstore.max_marginal_relevance_search(
                query, **self.search_kwargs
            )
        else:
            raise ValueError(f"search_type of {self.search_type} not allowed.")
        return docs
[docs]    def add_documents(self, documents: List[Document], **kwargs: Any) -> List[str]:
        """Add documents to vectorstore."""
        return self.vectorstore.add_documents(documents, **kwargs)
[docs]    async def aadd_documents(
        self, documents: List[Document], **kwargs: Any
    ) -> List[str]:
        """Add documents to vectorstore."""
        return await self.vectorstore.aadd_documents(documents, **kwargs)
Source code for langchain_community.vectorstores.redis.filters
from enum import Enum
from functools import wraps
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
from langchain_community.utilities.redis import TokenEscaper
# disable mypy error for dunder method overrides
# mypy: disable-error-code="override"
[docs]class RedisFilterOperator(Enum):
    """RedisFilterOperator enumerator is used to create RedisFilterExpressions."""
    EQ = 1
    NE = 2
    LT = 3
    GT = 4
    LE = 5
    GE = 6
    OR = 7
    AND = 8
    LIKE = 9
    IN = 10
[docs]class RedisFilter:
    """Collection of RedisFilterFields."""
[docs]    @staticmethod
    def text(field: str) -> "RedisText":
        return RedisText(field)
[docs]    @staticmethod
    def num(field: str) -> "RedisNum":
        return RedisNum(field)
[docs]    @staticmethod
    def tag(field: str) -> "RedisTag":
        return RedisTag(field)
[docs]class RedisFilterField:
    """Base class for RedisFilterFields."""
    escaper: "TokenEscaper" = TokenEscaper()
    OPERATORS: Dict[RedisFilterOperator, str] = {}
[docs]    def __init__(self, field: str):
        self._field = field
        self._value: Any = None
        self._operator: RedisFilterOperator = RedisFilterOperator.EQ
[docs]    def equals(self, other: "RedisFilterField") -> bool:
        if not isinstance(other, type(self)):
            return False
if not isinstance(other, type(self)):
            return False
        return self._field == other._field and self._value == other._value
    def _set_value(
        self, val: Any, val_type: Tuple[Any], operator: RedisFilterOperator
    ) -> None:
        # check that the operator is supported by this class
        if operator not in self.OPERATORS:
            raise ValueError(
                f"Operator {operator} not supported by {self.__class__.__name__}. "
                + f"Supported operators are {self.OPERATORS.values()}."
            )
        if not isinstance(val, val_type):
            raise TypeError(
                f"Right side argument passed to operator {self.OPERATORS[operator]} "
                f"with left side "
                f"argument {self.__class__.__name__} must be of type {val_type}, "
                f"received value {val}"
            )
        self._value = val
        self._operator = operator
[docs]def check_operator_misuse(func: Callable) -> Callable:
    """Decorator to check for misuse of equality operators."""
    @wraps(func)
    def wrapper(instance: Any, *args: Any, **kwargs: Any) -> Any:
        # Extracting 'other' from positional arguments or keyword arguments
        other = kwargs.get("other") if "other" in kwargs else None
        if not other:
            for arg in args:
                if isinstance(arg, type(instance)):
                    other = arg
                    break
        if isinstance(other, type(instance)):
            raise ValueError(
                "Equality operators are overridden for FilterExpression creation. Use "
                ".equals() for equality checks"
            )
        return func(instance, *args, **kwargs)
    return wrapper
)
        return func(instance, *args, **kwargs)
    return wrapper
[docs]class RedisTag(RedisFilterField):
    """A RedisFilterField representing a tag in a Redis index."""
    OPERATORS: Dict[RedisFilterOperator, str] = {
        RedisFilterOperator.EQ: "==",
        RedisFilterOperator.NE: "!=",
        RedisFilterOperator.IN: "==",
    }
    OPERATOR_MAP: Dict[RedisFilterOperator, str] = {
        RedisFilterOperator.EQ: "@%s:{%s}",
        RedisFilterOperator.NE: "(-@%s:{%s})",
        RedisFilterOperator.IN: "@%s:{%s}",
    }
    SUPPORTED_VAL_TYPES = (list, set, tuple, str, type(None))
[docs]    def __init__(self, field: str):
        """Create a RedisTag FilterField.
        Args:
            field (str): The name of the RedisTag field in the index to be queried
                against.
        """
        super().__init__(field)
    def _set_tag_value(
        self,
        other: Union[List[str], Set[str], Tuple[str], str],
        operator: RedisFilterOperator,
    ) -> None:
        if isinstance(other, (list, set, tuple)):
            try:
                # "if val" clause removes non-truthy values from list
                other = [str(val) for val in other if val]
            except ValueError:
                raise ValueError("All tags within collection must be strings")
        # above to catch the "" case
        elif not other:
            other = []
        elif isinstance(other, str):
            other = [other]
other = []
        elif isinstance(other, str):
            other = [other]
        self._set_value(other, self.SUPPORTED_VAL_TYPES, operator)  # type: ignore
    @check_operator_misuse
    def __eq__(
        self, other: Union[List[str], Set[str], Tuple[str], str]
    ) -> "RedisFilterExpression":
        """Create a RedisTag equality filter expression.
        Args:
            other (Union[List[str], Set[str], Tuple[str], str]):
                The tag(s) to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisTag
            >>> filter = RedisTag("brand") == "nike"
        """
        self._set_tag_value(other, RedisFilterOperator.EQ)
        return RedisFilterExpression(str(self))
    @check_operator_misuse
    def __ne__(
        self, other: Union[List[str], Set[str], Tuple[str], str]
    ) -> "RedisFilterExpression":
        """Create a RedisTag inequality filter expression.
        Args:
            other (Union[List[str], Set[str], Tuple[str], str]):
                The tag(s) to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisTag
            >>> filter = RedisTag("brand") != "nike"
        """
        self._set_tag_value(other, RedisFilterOperator.NE)
        return RedisFilterExpression(str(self))
    @property
    def _formatted_tag_value(self) -> str:
        return "|".join([self.escaper.escape(tag) for tag in self._value])
    def __str__(self) -> str:
        """Return the query syntax for a RedisTag filter expression."""
        if not self._value:
            return "*"
if not self._value:
            return "*"
        return self.OPERATOR_MAP[self._operator] % (
            self._field,
            self._formatted_tag_value,
        )
[docs]class RedisNum(RedisFilterField):
    """A RedisFilterField representing a numeric field in a Redis index."""
    OPERATORS: Dict[RedisFilterOperator, str] = {
        RedisFilterOperator.EQ: "==",
        RedisFilterOperator.NE: "!=",
        RedisFilterOperator.LT: "<",
        RedisFilterOperator.GT: ">",
        RedisFilterOperator.LE: "<=",
        RedisFilterOperator.GE: ">=",
    }
    OPERATOR_MAP: Dict[RedisFilterOperator, str] = {
        RedisFilterOperator.EQ: "@%s:[%s %s]",
        RedisFilterOperator.NE: "(-@%s:[%s %s])",
        RedisFilterOperator.GT: "@%s:[(%s +inf]",
        RedisFilterOperator.LT: "@%s:[-inf (%s]",
        RedisFilterOperator.GE: "@%s:[%s +inf]",
        RedisFilterOperator.LE: "@%s:[-inf %s]",
    }
    SUPPORTED_VAL_TYPES = (int, float, type(None))
    def __str__(self) -> str:
        """Return the query syntax for a RedisNum filter expression."""
        if self._value is None:
            return "*"
        if (
            self._operator == RedisFilterOperator.EQ
            or self._operator == RedisFilterOperator.NE
        ):
            return self.OPERATOR_MAP[self._operator] % (
                self._field,
                self._value,
                self._value,
            )
        else:
self._value,
                self._value,
            )
        else:
            return self.OPERATOR_MAP[self._operator] % (self._field, self._value)
    @check_operator_misuse
    def __eq__(self, other: Union[int, float]) -> "RedisFilterExpression":
        """Create a Numeric equality filter expression.
        Args:
            other (Union[int, float]): The value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisNum
            >>> filter = RedisNum("zipcode") == 90210
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.EQ)  # type: ignore
        return RedisFilterExpression(str(self))
    @check_operator_misuse
    def __ne__(self, other: Union[int, float]) -> "RedisFilterExpression":
        """Create a Numeric inequality filter expression.
        Args:
            other (Union[int, float]): The value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisNum
            >>> filter = RedisNum("zipcode") != 90210
        """
        self._set_value(other, self.SUPPORTED_VAL_TYPES, RedisFilterOperator.NE)  # type: ignore
        return RedisFilterExpression(str(self))
    def __gt__(self, other: Union[int, float]) -> "RedisFilterExpression":
        """Create a Numeric greater than filter expression.
        Args:
            other (Union[int, float]): The value to filter on.
        Example:
            >>> from langchain_community.vectorstores.redis import RedisNum
            >>> filter = RedisNum("age") > 18
        """