Repository: modin-project/modin

Issue:
[OmniSci] Support 'skew' aggregate for OmniSci backend



Discussion:

Resolving patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py b/modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py
--- a/modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py
+++ b/modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py
@@ -118,7 +118,80 @@ def gen_reduce_expr(self):
                 .pow(LiteralExpr(0.5))
             )
 
-    _compound_aggregates = {"std": StdAggregate}
+    class SkewAggregate(CompoundAggregate):
+        def __init__(self, builder, arg):
+            assert isinstance(arg, InputRefExpr)
+            super().__init__(builder, arg)
+
+            self._quad_name = self._arg.column + "__quad__"
+            self._cube_name = self._arg.column + "__cube__"
+            self._sum_name = self._arg.column + "__sum__"
+            self._quad_sum_name = self._arg.column + "__quad_sum__"
+            self._cube_sum_name = self._arg.column + "__cube_sum__"
+            self._count_name = self._arg.column + "__count__"
+
+        def gen_proj_exprs(self):
+            quad_expr = self._builder._translate(self._arg.mul(self._arg))
+            cube_expr = self._builder._translate(
+                self._arg.mul(self._arg).mul(self._arg)
+            )
+            return {self._quad_name: quad_expr, self._cube_name: cube_expr}
+
+        def gen_agg_exprs(self):
+            count_expr = self._builder._translate(AggregateExpr("count", self._arg))
+            sum_expr = self._builder._translate(AggregateExpr("sum", self._arg))
+            self._sum_dtype = sum_expr._dtype
+            qsum_expr = AggregateExpr(
+                "SUM",
+                self._builder._ref_idx(self._arg.modin_frame, self._quad_name),
+                dtype=sum_expr._dtype,
+            )
+            csum_expr = AggregateExpr(
+                "SUM",
+                self._builder._ref_idx(self._arg.modin_frame, self._cube_name),
+                dtype=sum_expr._dtype,
+            )
+
+            return {
+                self._sum_name: sum_expr,
+                self._quad_sum_name: qsum_expr,
+                self._cube_sum_name: csum_expr,
+                self._count_name: count_expr,
+            }
+
+        def gen_reduce_expr(self):
+            count_expr = self._builder._ref(self._arg.modin_frame, self._count_name)
+            count_expr._dtype = _get_dtype(int)
+            sum_expr = self._builder._ref(self._arg.modin_frame, self._sum_name)
+            sum_expr._dtype = self._sum_dtype
+            qsum_expr = self._builder._ref(self._arg.modin_frame, self._quad_sum_name)
+            qsum_expr._dtype = self._sum_dtype
+            csum_expr = self._builder._ref(self._arg.modin_frame, self._cube_sum_name)
+            csum_expr._dtype = self._sum_dtype
+
+            mean_expr = sum_expr.truediv(count_expr)
+
+            # n * sqrt(n - 1) / (n - 2)
+            #  * (sum(x ** 3) - 3 * mean * sum(x * x) + 2 * mean * mean * sum(x))
+            #  / (sum(x * x) - mean * sum(x)) ** 1.5
+            part1 = count_expr.mul(
+                count_expr.sub(LiteralExpr(1)).pow(LiteralExpr(0.5))
+            ).truediv(count_expr.sub(LiteralExpr(2)))
+            part2 = csum_expr.sub(mean_expr.mul(qsum_expr).mul(LiteralExpr(3.0))).add(
+                mean_expr.mul(mean_expr).mul(sum_expr).mul(LiteralExpr(2.0))
+            )
+            part3 = qsum_expr.sub(mean_expr.mul(sum_expr)).pow(LiteralExpr(1.5))
+            skew_expr = part1.mul(part2).truediv(part3)
+
+            # The result is NULL if n <= 2
+            return build_if_then_else(
+                count_expr.le(LiteralExpr(2)),
+                LiteralExpr(None),
+                skew_expr,
+                skew_expr._dtype,
+            )
+
+    _compound_aggregates = {"std": StdAggregate, "skew": SkewAggregate}
 
     class InputContext:
         _simple_aggregates = {
diff --git a/modin/experimental/engines/omnisci_on_ray/frame/expr.py b/modin/experimental/engines/omnisci_on_ray/frame/expr.py
--- a/modin/experimental/engines/omnisci_on_ray/frame/expr.py
+++ b/modin/experimental/engines/omnisci_on_ray/frame/expr.py
@@ -37,7 +37,7 @@ def _get_common_dtype(lhs_dtype, rhs_dtype):
 
 _aggs_preserving_numeric_type = {"sum", "min", "max"}
 _aggs_with_int_result = {"count", "size"}
-_aggs_with_float_result = {"mean", "std"}
+_aggs_with_float_result = {"mean", "std", "skew"}
 
 
 def _agg_dtype(agg, dtype):
@@ -83,6 +83,12 @@ def eq(self, other):
         new_expr = OpExpr("=", [self, other], _get_dtype(bool))
         return new_expr
 
+    def le(self, other):
+        if not isinstance(other, BaseExpr):
+            other = LiteralExpr(other)
+        new_expr = OpExpr("<=", [self, other], _get_dtype(bool))
+        return new_expr
+
     def cast(self, res_type):
         new_expr = OpExpr("CAST", [self], res_type)
         return new_expr


Test patch:
diff --git a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
--- a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
+++ b/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py
@@ -731,6 +731,23 @@ def std(df, **kwargs):
 
         run_and_compare(std, data=self.std_data, force_lazy=False)
 
+    skew_data = {
+        "a": [1, 2, 1, 1, 1, 2, 2, 2, 1, 2, 3, 4, 4],
+        "b": [4, 3, 1, 6, 9, 8, 0, 9, 5, 13, 12, 44, 6],
+        "c": [12.8, 45.6, 23.5, 12.4, 11.2, None, 56.4, 12.5, 1, 55, 4.5, 7.8, 9.4],
+    }
+
+    def test_agg_skew(self):
+        def std(df, **kwargs):
+            df = df.groupby("a").agg({"b": "skew", "c": "skew"})
+            if not isinstance(df, pandas.DataFrame):
+                df = to_pandas(df)
+            df["b"] = df["b"].apply(lambda x: round(x, 10))
+            df["c"] = df["c"].apply(lambda x: round(x, 10))
+            return df
+
+        run_and_compare(std, data=self.skew_data, force_lazy=False)
+
 
 class TestMerge:
     data = {