Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/dataset/sqlalchemy_dataset.py b/great_expectations/dataset/sqlalchemy_dataset.py
--- a/great_expectations/dataset/sqlalchemy_dataset.py
+++ b/great_expectations/dataset/sqlalchemy_dataset.py
@@ -1,10 +1,11 @@
 import inspect
+import itertools
 import logging
 import traceback
 import warnings
 from datetime import datetime
 from functools import wraps
-from typing import Dict, Iterable, List
+from typing import Dict, Iterable, List, Optional
 
 import numpy as np
 import pandas as pd
@@ -935,6 +936,11 @@ def get_column_quantiles(
                 quantiles=quantiles,
                 allow_relative_error=allow_relative_error,
             )
+        elif self.sql_engine_dialect.name.lower() == "sqlite":
+            return self._get_column_quantiles_sqlite(
+                column=column,
+                quantiles=quantiles,
+            )
         else:
             return convert_to_json_serializable(
                 self._get_column_quantiles_generic_sqlalchemy(
@@ -1050,6 +1056,59 @@ def _get_column_quantiles_mysql(self, column: str, quantiles: Iterable) -> list:
         except ProgrammingError as pe:
             self._treat_quantiles_exception(pe)
 
+    def _get_column_quantiles_sqlite(
+        self,
+        column,
+        quantiles: Iterable,
+    ) -> list:
+        """
+        The present implementation is somewhat inefficient, because it requires as many calls to "self.engine.execute()"
+        as the number of partitions in the "quantiles" parameter (albeit, typically, only a few).  However, this is the
+        only mechanism available for SQLite at the present time (11/17/2021), because the analytical processing is not a
+        very strongly represented capability of the SQLite database management system.
+        """
+        table_row_count_query: Select = sa.select([sa.func.count()]).select_from(
+            self._table
+        )
+        table_row_count_result: Optional[Row] = None
+        try:
+            table_row_count_result = self.engine.execute(
+                table_row_count_query
+            ).fetchone()
+        except ProgrammingError as pe:
+            self._treat_quantiles_exception(pe)
+
+        table_row_count: int
+        if table_row_count_result is not None:
+            table_row_count = list(table_row_count_result)[0]
+        else:
+            table_row_count = 0
+
+        offsets: List[int] = [quantile * table_row_count - 1 for quantile in quantiles]
+        quantile_queries: List[Select] = [
+            sa.select([sa.column(column)])
+            .order_by(sa.column(column).asc())
+            .offset(offset)
+            .limit(1)
+            .select_from(self._table)
+            for offset in offsets
+        ]
+
+        quantile_result: Row
+        quantile_query: Select
+        try:
+            quantiles_results: List[Row] = [
+                self.engine.execute(quantile_query).fetchone()
+                for quantile_query in quantile_queries
+            ]
+            return list(
+                itertools.chain.from_iterable(
+                    [list(quantile_result) for quantile_result in quantiles_results]
+                )
+            )
+        except ProgrammingError as pe:
+            self._treat_quantiles_exception(pe)
+
     # Support for computing the quantiles column for PostGreSQL and Redshift is included in the same method as that for
     # the generic sqlalchemy compatible DBMS engine, because users often use the postgresql driver to connect to Redshift
     # The key functional difference is that Redshift does not support the aggregate function
diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_quantile_values.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_quantile_values.py
--- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_quantile_values.py
+++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_quantile_values.py
@@ -1,15 +1,20 @@
+import itertools
 import logging
 import traceback
 from collections.abc import Iterable
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 import numpy as np
 
+from great_expectations.core import ExpectationConfiguration
 from great_expectations.execution_engine import (
     PandasExecutionEngine,
     SparkDFExecutionEngine,
 )
-from great_expectations.execution_engine.execution_engine import MetricDomainTypes
+from great_expectations.execution_engine.execution_engine import (
+    ExecutionEngine,
+    MetricDomainTypes,
+)
 from great_expectations.execution_engine.sqlalchemy_execution_engine import (
     SqlAlchemyExecutionEngine,
 )
@@ -23,6 +28,7 @@
 )
 from great_expectations.expectations.metrics.metric_provider import metric_value
 from great_expectations.expectations.metrics.util import attempt_allowing_relative_error
+from great_expectations.validator.metric_configuration import MetricConfiguration
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +44,7 @@
     ProgrammingError = None
     Select = None
     Label = None
-    TextClaus = None
+    TextClause = None
     WithinGroup = None
     CTE = None
 
@@ -79,7 +85,7 @@ def _pandas(cls, column, quantiles, allow_relative_error, **kwargs):
     @metric_value(engine=SqlAlchemyExecutionEngine)
     def _sqlalchemy(
         cls,
-        execution_engine: "SqlAlchemyExecutionEngine",
+        execution_engine: SqlAlchemyExecutionEngine,
         metric_domain_kwargs: Dict,
         metric_value_kwargs: Dict,
         metrics: Dict[Tuple, Any],
@@ -98,6 +104,7 @@ def _sqlalchemy(
         dialect = sqlalchemy_engine.dialect
         quantiles = metric_value_kwargs["quantiles"]
         allow_relative_error = metric_value_kwargs.get("allow_relative_error", False)
+        table_row_count = metrics.get("table.row_count")
         if dialect.name.lower() == "mssql":
             return _get_column_quantiles_mssql(
                 column=column,
@@ -136,6 +143,14 @@ def _sqlalchemy(
                 selectable=selectable,
                 sqlalchemy_engine=sqlalchemy_engine,
             )
+        elif dialect.name.lower() == "sqlite":
+            return _get_column_quantiles_sqlite(
+                column=column,
+                quantiles=quantiles,
+                selectable=selectable,
+                sqlalchemy_engine=sqlalchemy_engine,
+                table_row_count=table_row_count,
+            )
         else:
             return _get_column_quantiles_generic_sqlalchemy(
                 column=column,
@@ -149,7 +164,7 @@ def _sqlalchemy(
     @metric_value(engine=SparkDFExecutionEngine)
     def _spark(
         cls,
-        execution_engine: "SqlAlchemyExecutionEngine",
+        execution_engine: SqlAlchemyExecutionEngine,
         metric_domain_kwargs: Dict,
         metric_value_kwargs: Dict,
         metrics: Dict[Tuple, Any],
@@ -177,6 +192,33 @@ def _spark(
             )
         return df.approxQuantile(column, list(quantiles), allow_relative_error)
 
+    @classmethod
+    def _get_evaluation_dependencies(
+        cls,
+        metric: MetricConfiguration,
+        configuration: Optional[ExpectationConfiguration] = None,
+        execution_engine: Optional[ExecutionEngine] = None,
+        runtime_configuration: Optional[dict] = None,
+    ):
+        dependencies: dict = super()._get_evaluation_dependencies(
+            metric=metric,
+            configuration=configuration,
+            execution_engine=execution_engine,
+            runtime_configuration=runtime_configuration,
+        )
+
+        table_domain_kwargs: dict = {
+            k: v for k, v in metric.metric_domain_kwargs.items() if k != "column"
+        }
+        dependencies["table.row_count"] = MetricConfiguration(
+            metric_name="table.row_count",
+            metric_domain_kwargs=table_domain_kwargs,
+            metric_value_kwargs=None,
+            metric_dependencies=None,
+        )
+
+        return dependencies
+
 
 def _get_column_quantiles_mssql(
     column, quantiles: Iterable, selectable, sqlalchemy_engine
@@ -282,6 +324,47 @@ def _get_column_quantiles_mysql(
         raise pe
 
 
+def _get_column_quantiles_sqlite(
+    column, quantiles: Iterable, selectable, sqlalchemy_engine, table_row_count
+) -> list:
+    """
+    The present implementation is somewhat inefficient, because it requires as many
+    calls to "sqlalchemy_engine.execute()" as the number of partitions in the "quantiles" parameter (albeit, typically,
+    only a few).  However, this is the only mechanism available for SQLite at the present time (11/17/2021), because
+    the analytical processing is not a very strongly represented capability of the SQLite database management system.
+    """
+    offsets: List[int] = [quantile * table_row_count - 1 for quantile in quantiles]
+    quantile_queries: List[Select] = [
+        sa.select([column])
+        .order_by(column.asc())
+        .offset(offset)
+        .limit(1)
+        .select_from(selectable)
+        for offset in offsets
+    ]
+
+    quantile_result: Row
+    quantile_query: Select
+    try:
+        quantiles_results: List[Row] = [
+            sqlalchemy_engine.execute(quantile_query).fetchone()
+            for quantile_query in quantile_queries
+        ]
+        return list(
+            itertools.chain.from_iterable(
+                [list(quantile_result) for quantile_result in quantiles_results]
+            )
+        )
+    except ProgrammingError as pe:
+        exception_message: str = "An SQL syntax Exception occurred."
+        exception_traceback: str = traceback.format_exc()
+        exception_message += (
+            f'{type(pe).__name__}: "{str(pe)}".  Traceback: "{exception_traceback}".'
+        )
+        logger.error(exception_message)
+        raise pe
+
+
 # Support for computing the quantiles column for PostGreSQL and Redshift is included in the same method as that for
 # the generic sqlalchemy compatible DBMS engine, because users often use the postgresql driver to connect to Redshift
 # The key functional difference is that Redshift does not support the aggregate function


Test patch:
diff --git a/tests/expectations/metrics/test_core.py b/tests/expectations/metrics/test_core.py
--- a/tests/expectations/metrics/test_core.py
+++ b/tests/expectations/metrics/test_core.py
@@ -110,6 +110,120 @@ def test_stdev_metric_pd():
     assert results == {desired_metric.id: 1}
 
 
+def test_quantiles_metric_pd():
+    engine = build_pandas_engine(pd.DataFrame({"a": [1, 2, 3, 4]}))
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    table_columns_metric, results = get_table_columns_metric(engine=engine)
+    metrics.update(results)
+
+    desired_metric = MetricConfiguration(
+        metric_name="column.quantile_values",
+        metric_domain_kwargs={"column": "a"},
+        metric_value_kwargs={
+            "quantiles": [2.5e-1, 5.0e-1, 7.5e-1],
+            "allow_relative_error": "linear",
+        },
+        metric_dependencies={
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(desired_metric,), metrics=metrics
+    )
+    metrics.update(results)
+    assert results == {desired_metric.id: [1.75, 2.5, 3.25]}
+
+
+def test_quantiles_metric_sa(sa):
+    engine = build_sa_engine(pd.DataFrame({"a": [1, 2, 3, 4]}), sa)
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    table_columns_metric, results = get_table_columns_metric(engine=engine)
+    metrics.update(results)
+
+    partial_metric = MetricConfiguration(
+        metric_name="table.row_count.aggregate_fn",
+        metric_domain_kwargs={},
+        metric_value_kwargs=None,
+    )
+
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(partial_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    table_row_count_metric = MetricConfiguration(
+        metric_name="table.row_count",
+        metric_domain_kwargs={},
+        metric_value_kwargs=None,
+        metric_dependencies={
+            "metric_partial_fn": partial_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(table_row_count_metric,), metrics=metrics
+    )
+    metrics.update(results)
+
+    desired_metric = MetricConfiguration(
+        metric_name="column.quantile_values",
+        metric_domain_kwargs={"column": "a"},
+        metric_value_kwargs={
+            "quantiles": [2.5e-1, 5.0e-1, 7.5e-1],
+        },
+        metric_dependencies={
+            "table.columns": table_columns_metric,
+            "table.row_count": table_row_count_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(desired_metric,), metrics=metrics
+    )
+    metrics.update(results)
+    assert results == {desired_metric.id: [1.0, 2.0, 3.0]}
+
+
+def test_quantiles_metric_spark(spark_session):
+    engine: SparkDFExecutionEngine = build_spark_engine(
+        spark=spark_session,
+        df=pd.DataFrame({"a": [1, 2, 3, 4]}),
+        batch_id="my_id",
+    )
+
+    metrics: dict = {}
+
+    table_columns_metric: MetricConfiguration
+    results: dict
+
+    table_columns_metric, results = get_table_columns_metric(engine=engine)
+    metrics.update(results)
+
+    desired_metric = MetricConfiguration(
+        metric_name="column.quantile_values",
+        metric_domain_kwargs={"column": "a"},
+        metric_value_kwargs={
+            "quantiles": [2.5e-1, 5.0e-1, 7.5e-1],
+        },
+        metric_dependencies={
+            "table.columns": table_columns_metric,
+        },
+    )
+    results = engine.resolve_metrics(
+        metrics_to_resolve=(desired_metric,), metrics=metrics
+    )
+    metrics.update(results)
+    assert results == {desired_metric.id: [1.0, 2.0, 3.0]}
+
+
 def test_max_metric_column_exists_pd():
     df = pd.DataFrame({"a": [1, 2, 3, 3, None]})
     batch = Batch(data=df)
@@ -366,7 +480,9 @@ def test_map_value_set_sa(sa):
         metric_name="column_values.in_set.unexpected_count",
         metric_domain_kwargs={"column": "a"},
         metric_value_kwargs={"value_set": [1, 2, 3]},
-        metric_dependencies={"metric_partial_fn": aggregate_partial},
+        metric_dependencies={
+            "metric_partial_fn": aggregate_partial,
+        },
     )
     results = engine.resolve_metrics(
         metrics_to_resolve=(desired_metric,), metrics=metrics
@@ -440,7 +556,9 @@ def test_map_value_set_spark(spark_session, basic_spark_df_execution_engine):
         metric_name="column_values.in_set.unexpected_count",
         metric_domain_kwargs={"column": "a"},
         metric_value_kwargs={"value_set": [1, 2, 3]},
-        metric_dependencies={"metric_partial_fn": aggregate_partial},
+        metric_dependencies={
+            "metric_partial_fn": aggregate_partial,
+        },
     )
 
     results = engine.resolve_metrics(
@@ -484,7 +602,9 @@ def test_map_value_set_spark(spark_session, basic_spark_df_execution_engine):
         metric_name="column_values.in_set.unexpected_count",
         metric_domain_kwargs={"column": "a"},
         metric_value_kwargs={"value_set": [1, 2, 3]},
-        metric_dependencies={"metric_partial_fn": aggregate_partial},
+        metric_dependencies={
+            "metric_partial_fn": aggregate_partial,
+        },
     )
 
     results = engine.resolve_metrics(
@@ -1389,7 +1509,9 @@ def test_z_score_under_threshold_spark(spark_session):
         metric_name="column.mean",
         metric_domain_kwargs={"column": "a"},
         metric_value_kwargs=None,
-        metric_dependencies={"metric_partial_fn": mean},
+        metric_dependencies={
+            "metric_partial_fn": mean,
+        },
     )
     stdev = MetricConfiguration(
         metric_name="column.standard_deviation",
@@ -1449,7 +1571,9 @@ def test_z_score_under_threshold_spark(spark_session):
         metric_name="column_values.z_score.under_threshold.unexpected_count",
         metric_domain_kwargs={"column": "a"},
         metric_value_kwargs={"double_sided": True, "threshold": 2},
-        metric_dependencies={"metric_partial_fn": desired_metric},
+        metric_dependencies={
+            "metric_partial_fn": desired_metric,
+        },
     )
     results = engine.resolve_metrics(
         metrics_to_resolve=(desired_metric,), metrics=metrics
@@ -1702,7 +1826,9 @@ def test_table_metric_sa(sa):
         metric_name="table.row_count",
         metric_domain_kwargs={},
         metric_value_kwargs=None,
-        metric_dependencies={"metric_partial_fn": desired_metric},
+        metric_dependencies={
+            "metric_partial_fn": desired_metric,
+        },
     )
     results = engine.resolve_metrics(
         metrics_to_resolve=(desired_metric,), metrics=results
diff --git a/tests/profile/test_basic_suite_builder_profiler.py b/tests/profile/test_basic_suite_builder_profiler.py
--- a/tests/profile/test_basic_suite_builder_profiler.py
+++ b/tests/profile/test_basic_suite_builder_profiler.py
@@ -134,6 +134,7 @@ def test__create_expectations_for_numeric_column(
     if test_backend in [
         "PandasDataset",
         "SparkDFDataset",
+        "sqlite",
         "postgresql",
         "mysql",
         "mssql",
@@ -162,6 +163,7 @@ def test__create_expectations_for_numeric_column(
             "expect_column_max_to_be_between",
             "expect_column_mean_to_be_between",
             "expect_column_median_to_be_between",
+            "expect_column_quantile_values_to_be_between",
             "expect_column_values_to_not_be_null",
         }
 
diff --git a/tests/test_definitions/column_aggregate_expectations/expect_column_quantile_values_to_be_between.json b/tests/test_definitions/column_aggregate_expectations/expect_column_quantile_values_to_be_between.json
--- a/tests/test_definitions/column_aggregate_expectations/expect_column_quantile_values_to_be_between.json
+++ b/tests/test_definitions/column_aggregate_expectations/expect_column_quantile_values_to_be_between.json
@@ -35,8 +35,7 @@
             }
           },
           "tolerance": 0.1,
-          "_note": "The large tolerance here documents implementation differences between pandas, sql, and spark wrt interpolation behavior / specific ntile calculation",
-          "suppress_test_for": ["sqlite"]
+          "_note": "The large tolerance here documents implementation differences between pandas, sql, and spark wrt interpolation behavior / specific ntile calculation"
         },
         {
           "title": "basic_positive_test_normal_quartiles",
@@ -56,8 +55,7 @@
             }
           },
           "tolerance": 0.1,
-          "_note": "The large tolerance here documents implementation differences between pandas, sql, and spark wrt interpolation behavior / specific ntile calculation",
-          "suppress_test_for": ["sqlite"]
+          "_note": "The large tolerance here documents implementation differences between pandas, sql, and spark wrt interpolation behavior / specific ntile calculation"
         },
         {
           "title": "basic_positive_test_uneven_spacing",
@@ -76,7 +74,6 @@
               "values": [-3.40196868, -1.72089571, -0.70115633, -0.04059954,  0.62130846, 1.6855355 ,  3.58540782]
             }
           },
-          "suppress_test_for": ["sqlite"],
           "tolerance": 0.1,
           "_note": "The large tolerance here documents implementation differences between pandas, sql, and spark wrt interpolation behavior / specific ntile calculation"
       },
@@ -92,8 +89,7 @@
           },
           "out": {
             "success": false
-          },
-          "suppress_test_for": ["sqlite"]
+          }
         }
     ]
   }]