viruPVS/langchain-docs huggingface dataset:
Source code for langchain.retrievers.self_query.dashvector
"""Logic for converting internal query language to a valid DashVector query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
[docs]class DashvectorTranslator(Visitor):
    """Logic for converting internal query language elements to valid filters."""
    allowed_operators = [Operator.AND, Operator.OR]
    allowed_comparators = [
        Comparator.EQ,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
        Comparator.LIKE,
    ]
    map_dict = {
        Operator.AND: " AND ",
        Operator.OR: " OR ",
        Comparator.EQ: " = ",
        Comparator.GT: " > ",
        Comparator.GTE: " >= ",
        Comparator.LT: " < ",
        Comparator.LTE: " <= ",
        Comparator.LIKE: " LIKE ",
    }
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        return self.map_dict[func]
[docs]    def visit_operation(self, operation: Operation) -> str:
        args = [arg.accept(self) for arg in operation.arguments]
        return self._format_func(operation.operator).join(args)
[docs]    def visit_comparison(self, comparison: Comparison) -> str:
        value = comparison.value
        if isinstance(value, str):
            if comparison.comparator == Comparator.LIKE:
                value = f"'%{value}%'"
            else:
                value = f"'{value}'"
        return (
else:
                value = f"'{value}'"
        return (
            f"{comparison.attribute}{self._format_func(comparison.comparator)}{value}"
        )
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"filter": structured_query.filter.accept(self)}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.supabase
from typing import Any, Dict, Tuple
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
[docs]class SupabaseVectorTranslator(Visitor):
    """Translate Langchain filters to Supabase PostgREST filters."""
    allowed_operators = [Operator.AND, Operator.OR]
    """Subset of allowed logical operators."""
    allowed_comparators = [
        Comparator.EQ,
        Comparator.NE,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
        Comparator.LIKE,
    ]
    """Subset of allowed logical comparators."""
    metadata_column = "metadata"
    def _map_comparator(self, comparator: Comparator) -> str:
        """
        Maps Langchain comparator to PostgREST comparator:
        https://postgrest.org/en/stable/references/api/tables_views.html#operators
        """
        postgrest_comparator = {
            Comparator.EQ: "eq",
            Comparator.NE: "neq",
            Comparator.GT: "gt",
            Comparator.GTE: "gte",
            Comparator.LT: "lt",
            Comparator.LTE: "lte",
            Comparator.LIKE: "like",
        }.get(comparator)
        if postgrest_comparator is None:
            raise Exception(
                f"Comparator '{comparator}' is not currently "
                "supported in Supabase Vector"
            )
        return postgrest_comparator
    def _get_json_operator(self, value: Any) -> str:
        if isinstance(value, str):
            return "->>"
if isinstance(value, str):
            return "->>"
        else:
            return "->"
[docs]    def visit_operation(self, operation: Operation) -> str:
        args = [arg.accept(self) for arg in operation.arguments]
        return f"{operation.operator.value}({','.join(args)})"
[docs]    def visit_comparison(self, comparison: Comparison) -> str:
        if isinstance(comparison.value, list):
            return self.visit_operation(
                Operation(
                    operator=Operator.AND,
                    arguments=(
                        Comparison(
                            comparator=comparison.comparator,
                            attribute=comparison.attribute,
                            value=value,
                        )
                        for value in comparison.value
                    ),
                )
            )
        return ".".join(
            [
                f"{self.metadata_column}{self._get_json_operator(comparison.value)}{comparison.attribute}",
                f"{self._map_comparator(comparison.comparator)}",
                f"{comparison.value}",
            ]
        )
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, Dict[str, str]]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"postgrest_filter": structured_query.filter.accept(self)}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.elasticsearch
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
[docs]class ElasticsearchTranslator(Visitor):
    """Translate `Elasticsearch` internal query language elements to valid filters."""
    allowed_comparators = [
        Comparator.EQ,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
        Comparator.CONTAIN,
        Comparator.LIKE,
    ]
    """Subset of allowed logical comparators."""
    allowed_operators = [Operator.AND, Operator.OR, Operator.NOT]
    """Subset of allowed logical operators."""
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        map_dict = {
            Operator.OR: "should",
            Operator.NOT: "must_not",
            Operator.AND: "must",
            Comparator.EQ: "term",
            Comparator.GT: "gt",
            Comparator.GTE: "gte",
            Comparator.LT: "lt",
            Comparator.LTE: "lte",
            Comparator.CONTAIN: "match",
            Comparator.LIKE: "match",
        }
        return map_dict[func]
[docs]    def visit_operation(self, operation: Operation) -> Dict:
        args = [arg.accept(self) for arg in operation.arguments]
        return {"bool": {self._format_func(operation.operator): args}}
[docs]    def visit_comparison(self, comparison: Comparison) -> Dict:
        # ElasticsearchStore filters require to target
        # the metadata object field
        field = f"metadata.{comparison.attribute}"
# the metadata object field
        field = f"metadata.{comparison.attribute}"
        is_range_comparator = comparison.comparator in [
            Comparator.GT,
            Comparator.GTE,
            Comparator.LT,
            Comparator.LTE,
        ]
        if is_range_comparator:
            return {
                "range": {
                    field: {self._format_func(comparison.comparator): comparison.value}
                }
            }
        if comparison.comparator == Comparator.CONTAIN:
            return {
                self._format_func(comparison.comparator): {
                    field: {"query": comparison.value}
                }
            }
        if comparison.comparator == Comparator.LIKE:
            return {
                self._format_func(comparison.comparator): {
                    field: {"query": comparison.value, "fuzziness": "AUTO"}
                }
            }
        # we assume that if the value is a string,
        # we want to use the keyword field
        field = f"{field}.keyword" if isinstance(comparison.value, str) else field
        return {self._format_func(comparison.comparator): {field: comparison.value}}
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"filter": [structured_query.filter.accept(self)]}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.mongodb_atlas
"""Logic for converting internal query language to a valid MongoDB Atlas query."""
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
MULTIPLE_ARITY_COMPARATORS = [Comparator.IN, Comparator.NIN]
[docs]class MongoDBAtlasTranslator(Visitor):
    """Translate Mongo internal query language elements to valid filters."""
    """Subset of allowed logical comparators."""
    allowed_comparators = [
        Comparator.EQ,
        Comparator.NE,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
        Comparator.IN,
        Comparator.NIN,
    ]
    """Subset of allowed logical operators."""
    allowed_operators = [Operator.AND, Operator.OR]
    ## Convert a operator or a comparator to Mongo Query Format
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        map_dict = {
            Operator.AND: "$and",
            Operator.OR: "$or",
            Comparator.EQ: "$eq",
            Comparator.NE: "$ne",
            Comparator.GTE: "$gte",
            Comparator.LTE: "$lte",
            Comparator.LT: "$lt",
            Comparator.GT: "$gt",
            Comparator.IN: "$in",
            Comparator.NIN: "$nin",
        }
        return map_dict[func]
[docs]    def visit_operation(self, operation: Operation) -> Dict:
        args = [arg.accept(self) for arg in operation.arguments]
        return {self._format_func(operation.operator): args}
return {self._format_func(operation.operator): args}
[docs]    def visit_comparison(self, comparison: Comparison) -> Dict:
        if comparison.comparator in MULTIPLE_ARITY_COMPARATORS and not isinstance(
            comparison.value, list
        ):
            comparison.value = [comparison.value]
        comparator = self._format_func(comparison.comparator)
        attribute = comparison.attribute
        return {attribute: {comparator: comparison.value}}
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"pre_filter": structured_query.filter.accept(self)}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.deeplake
"""Logic for converting internal query language to a valid Chroma query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
COMPARATOR_TO_TQL = {
    Comparator.EQ: "==",
    Comparator.GT: ">",
    Comparator.GTE: ">=",
    Comparator.LT: "<",
    Comparator.LTE: "<=",
}
OPERATOR_TO_TQL = {
    Operator.AND: "and",
    Operator.OR: "or",
    Operator.NOT: "NOT",
}
[docs]def can_cast_to_float(string: str) -> bool:
    """Check if a string can be cast to a float."""
    try:
        float(string)
        return True
    except ValueError:
        return False
[docs]class DeepLakeTranslator(Visitor):
    """Translate `DeepLake` internal query language elements to valid filters."""
    allowed_operators = [Operator.AND, Operator.OR, Operator.NOT]
    """Subset of allowed logical operators."""
    allowed_comparators = [
        Comparator.EQ,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
    ]
    """Subset of allowed logical comparators."""
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        if isinstance(func, Operator):
            value = OPERATOR_TO_TQL[func.value]  # type: ignore
        elif isinstance(func, Comparator):
            value = COMPARATOR_TO_TQL[func.value]  # type: ignore
value = COMPARATOR_TO_TQL[func.value]  # type: ignore
        return f"{value}"
[docs]    def visit_operation(self, operation: Operation) -> str:
        args = [arg.accept(self) for arg in operation.arguments]
        operator = self._format_func(operation.operator)
        return "(" + (" " + operator + " ").join(args) + ")"
[docs]    def visit_comparison(self, comparison: Comparison) -> str:
        comparator = self._format_func(comparison.comparator)
        values = comparison.value
        if isinstance(values, list):
            tql = []
            for value in values:
                comparison.value = value
                tql.append(self.visit_comparison(comparison))
            return "(" + (" or ").join(tql) + ")"
        if not can_cast_to_float(comparison.value):
            values = f"'{values}'"
        return f"metadata['{comparison.attribute}'] {comparator} {values}"
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            tqL = f"SELECT * WHERE {structured_query.filter.accept(self)}"
            kwargs = {"tql": tqL}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.redis
from __future__ import annotations
from typing import Any, Tuple
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
from langchain.vectorstores.redis import Redis
from langchain.vectorstores.redis.filters import (
    RedisFilterExpression,
    RedisFilterField,
    RedisFilterOperator,
    RedisNum,
    RedisTag,
    RedisText,
)
from langchain.vectorstores.redis.schema import RedisModel
_COMPARATOR_TO_BUILTIN_METHOD = {
    Comparator.EQ: "__eq__",
    Comparator.NE: "__ne__",
    Comparator.LT: "__lt__",
    Comparator.GT: "__gt__",
    Comparator.LTE: "__le__",
    Comparator.GTE: "__ge__",
    Comparator.CONTAIN: "__eq__",
    Comparator.LIKE: "__mod__",
}
[docs]class RedisTranslator(Visitor):
    """Translate"""
    allowed_comparators = (
        Comparator.EQ,
        Comparator.NE,
        Comparator.LT,
        Comparator.LTE,
        Comparator.GT,
        Comparator.GTE,
        Comparator.CONTAIN,
        Comparator.LIKE,
    )
    """Subset of allowed logical comparators."""
    allowed_operators = (Operator.AND, Operator.OR)
    """Subset of allowed logical operators."""
[docs]    def __init__(self, schema: RedisModel) -> None:
        self._schema = schema
    def _attribute_to_filter_field(self, attribute: str) -> RedisFilterField:
        if attribute in [tf.name for tf in self._schema.text]:
            return RedisText(attribute)
return RedisText(attribute)
        elif attribute in [tf.name for tf in self._schema.tag or []]:
            return RedisTag(attribute)
        elif attribute in [tf.name for tf in self._schema.numeric or []]:
            return RedisNum(attribute)
        else:
            raise ValueError(
                f"Invalid attribute {attribute} not in vector store schema. Schema is:"
                f"\n{self._schema.as_dict()}"
            )
[docs]    def visit_comparison(self, comparison: Comparison) -> RedisFilterExpression:
        filter_field = self._attribute_to_filter_field(comparison.attribute)
        comparison_method = _COMPARATOR_TO_BUILTIN_METHOD[comparison.comparator]
        return getattr(filter_field, comparison_method)(comparison.value)
[docs]    def visit_operation(self, operation: Operation) -> Any:
        left = operation.arguments[0].accept(self)
        if len(operation.arguments) > 2:
            right = self.visit_operation(
                Operation(
                    operator=operation.operator, arguments=operation.arguments[1:]
                )
            )
        else:
            right = operation.arguments[1].accept(self)
        redis_operator = (
            RedisFilterOperator.OR
            if operation.operator == Operator.OR
            else RedisFilterOperator.AND
        )
        return RedisFilterExpression(operator=redis_operator, left=left, right=right)
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"filter": structured_query.filter.accept(self)}
        return structured_query.query, kwargs
[docs]    @classmethod
    def from_vectorstore(cls, vectorstore: Redis) -> RedisTranslator:
Source code for langchain.retrievers.self_query.chroma
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
[docs]class ChromaTranslator(Visitor):
    """Translate `Chroma` internal query language elements to valid filters."""
    allowed_operators = [Operator.AND, Operator.OR]
    """Subset of allowed logical operators."""
    allowed_comparators = [
        Comparator.EQ,
        Comparator.NE,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
    ]
    """Subset of allowed logical comparators."""
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        return f"${func.value}"
[docs]    def visit_operation(self, operation: Operation) -> Dict:
        args = [arg.accept(self) for arg in operation.arguments]
        return {self._format_func(operation.operator): args}
[docs]    def visit_comparison(self, comparison: Comparison) -> Dict:
        return {
            comparison.attribute: {
                self._format_func(comparison.comparator): comparison.value
            }
        }
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"filter": structured_query.filter.accept(self)}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.milvus
"""Logic for converting internal query language to a valid Milvus query."""
from typing import Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
COMPARATOR_TO_BER = {
    Comparator.EQ: "==",
    Comparator.GT: ">",
    Comparator.GTE: ">=",
    Comparator.LT: "<",
    Comparator.LTE: "<=",
}
UNARY_OPERATORS = [Operator.NOT]
[docs]def process_value(value: Union[int, float, str]) -> str:
    """Convert a value to a string and add double quotes if it is a string.
    It required for comparators involving strings.
    Args:
        value: The value to convert.
    Returns:
        The converted value as a string.
    """
    #
    if isinstance(value, str):
        # If the value is already a string, add double quotes
        return f'"{value}"'
    else:
        # If the valueis not a string, convert it to a string without double quotes
        return str(value)
[docs]class MilvusTranslator(Visitor):
    """Translate Milvus internal query language elements to valid filters."""
    """Subset of allowed logical operators."""
    allowed_operators = [Operator.AND, Operator.NOT, Operator.OR]
    """Subset of allowed logical comparators."""
    allowed_comparators = [
        Comparator.EQ,
        Comparator.GT,
        Comparator.GTE,
        Comparator.LT,
        Comparator.LTE,
    ]
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        value = func.value
        if isinstance(func, Comparator):
            value = COMPARATOR_TO_BER[func]
        return f"{value}"
[docs]    def visit_operation(self, operation: Operation) -> str:
        if operation.operator in UNARY_OPERATORS and len(operation.arguments) == 1:
            operator = self._format_func(operation.operator)
            return operator + "(" + operation.arguments[0].accept(self) + ")"
        elif operation.operator in UNARY_OPERATORS:
            raise ValueError(
                f'"{operation.operator.value}" can have only one argument in Milvus'
            )
        else:
            args = [arg.accept(self) for arg in operation.arguments]
            operator = self._format_func(operation.operator)
            return "(" + (" " + operator + " ").join(args) + ")"
[docs]    def visit_comparison(self, comparison: Comparison) -> str:
        comparator = self._format_func(comparison.comparator)
        processed_value = process_value(comparison.value)
        attribute = comparison.attribute
        return "( " + attribute + " " + comparator + " " + processed_value + " )"
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else:
            kwargs = {"expr": structured_query.filter.accept(self)}
        return structured_query.query, kwargs
Source code for langchain.retrievers.self_query.pinecone
from typing import Dict, Tuple, Union
from langchain.chains.query_constructor.ir import (
    Comparator,
    Comparison,
    Operation,
    Operator,
    StructuredQuery,
    Visitor,
)
[docs]class PineconeTranslator(Visitor):
    """Translate `Pinecone` internal query language elements to valid filters."""
    allowed_comparators = (
        Comparator.EQ,
        Comparator.NE,
        Comparator.LT,
        Comparator.LTE,
        Comparator.GT,
        Comparator.GTE,
        Comparator.IN,
        Comparator.NIN,
    )
    """Subset of allowed logical comparators."""
    allowed_operators = (Operator.AND, Operator.OR)
    """Subset of allowed logical operators."""
    def _format_func(self, func: Union[Operator, Comparator]) -> str:
        self._validate_func(func)
        return f"${func.value}"
[docs]    def visit_operation(self, operation: Operation) -> Dict:
        args = [arg.accept(self) for arg in operation.arguments]
        return {self._format_func(operation.operator): args}
[docs]    def visit_comparison(self, comparison: Comparison) -> Dict:
        if comparison.comparator in (Comparator.IN, Comparator.NIN) and not isinstance(
            comparison.value, list
        ):
            comparison.value = [comparison.value]
        return {
            comparison.attribute: {
                self._format_func(comparison.comparator): comparison.value
            }
        }
[docs]    def visit_structured_query(
        self, structured_query: StructuredQuery
    ) -> Tuple[str, dict]:
        if structured_query.filter is None:
            kwargs = {}
        else: