Repository: NVIDIA-Merlin/NVTabular

Issue:
[FEA] Normalize for list
**Is your feature request related to a problem? Please describe.**
As a user, I want to be able to use the Normalize on a columns with list values.

**Describe the solution you'd like**
Example dataset:
```
df = cudf.DataFrame({'userid': ['a', 'b', 'c'], 'col': [[10,100,1000], [10,5], [50,10]]})

normalize_col = ['col'] >> nvt.ops.Normalize()
```

The Normalize op will collect the mean/std over all values in the column `col`, when calling fit. It does not depend on the position in the list.

The normalize op will transform the column `col` and the lists contain normalized values.



Discussion:

Resolving patch:
diff --git a/nvtabular/ops/moments.py b/nvtabular/ops/moments.py
--- a/nvtabular/ops/moments.py
+++ b/nvtabular/ops/moments.py
@@ -22,6 +22,8 @@
 from dask.delayed import Delayed
 from dask.highlevelgraph import HighLevelGraph
 
+from nvtabular.dispatch import _flatten_list_column_values, _is_list_dtype
+
 
 def _custom_moments(ddf, split_every=32):
 
@@ -62,23 +64,24 @@ def _custom_moments(ddf, split_every=32):
 
 
 def _chunkwise_moments(df):
-    df2 = type(df)()
-    for col in df.columns:
-        df2[col] = df[col].astype("float64").pow(2)
-    vals = {
-        "df-count": df.count().to_frame().transpose(),
-        "df-sum": df.sum().astype("float64").to_frame().transpose(),
-        "df2-sum": df2.sum().to_frame().transpose(),
-    }
+    vals = {name: type(df)() for name in ["count", "sum", "squaredsum"]}
+    for name in df.columns:
+        column = df[name]
+        if _is_list_dtype(column):
+            column = _flatten_list_column_values(column)
+
+        vals["count"][name] = [column.count()]
+        vals["sum"][name] = [column.sum().astype("float64")]
+        vals["squaredsum"][name] = [column.astype("float64").pow(2).sum()]
+
     # NOTE: Perhaps we should convert to pandas here
     # (since we know the results should be small)?
-    del df2
     return vals
 
 
 def _tree_node_moments(inputs):
     out = {}
-    for val in ["df-count", "df-sum", "df2-sum"]:
+    for val in ["count", "sum", "squaredsum"]:
         df_list = [x.get(val, None) for x in inputs]
         df_list = [df for df in df_list if df is not None]
         out[val] = _concat(df_list, ignore_index=True).sum().to_frame().transpose()
@@ -86,9 +89,9 @@ def _tree_node_moments(inputs):
 
 
 def _finalize_moments(inp, ddof=1):
-    n = inp["df-count"].iloc[0]
-    x = inp["df-sum"].iloc[0]
-    x2 = inp["df2-sum"].iloc[0]
+    n = inp["count"].iloc[0]
+    x = inp["sum"].iloc[0]
+    x2 = inp["squaredsum"].iloc[0]
     if hasattr(n, "to_pandas"):
         n = n.to_pandas()
         x = x.to_pandas()
@@ -105,7 +108,7 @@ def _finalize_moments(inp, ddof=1):
     var[(n - ddof) == 0] = np.nan
 
     # Construct output DataFrame
-    out = pd.DataFrame(index=inp["df-count"].columns)
+    out = pd.DataFrame(index=inp["count"].columns)
     out["count"] = n
     out["sum"] = x
     out["sum2"] = x2
diff --git a/nvtabular/ops/normalize.py b/nvtabular/ops/normalize.py
--- a/nvtabular/ops/normalize.py
+++ b/nvtabular/ops/normalize.py
@@ -16,7 +16,13 @@
 import dask.dataframe as dd
 import numpy
 
-from ..dispatch import DataFrameType, annotate
+from ..dispatch import (
+    DataFrameType,
+    _encode_list_column,
+    _flatten_list_column_values,
+    _is_list_dtype,
+    annotate,
+)
 from ..tags import Tags
 from .moments import _custom_moments
 from .operator import ColumnSelector, Operator, Supports
@@ -58,11 +64,22 @@ def fit_finalize(self, dask_stats):
     def transform(self, col_selector: ColumnSelector, df: DataFrameType) -> DataFrameType:
         new_df = type(df)()
         for name in col_selector.names:
+            values = df[name]
+            list_col = _is_list_dtype(values)
+            if list_col:
+                values = _flatten_list_column_values(values)
+
             if self.stds[name] > 0:
-                new_df[name] = (df[name] - self.means[name]) / (self.stds[name])
+                values = (values - self.means[name]) / (self.stds[name])
             else:
-                new_df[name] = df[name] - self.means[name]
-            new_df[name] = new_df[name].astype("float32")
+                values = values - self.means[name]
+
+            values = values.astype("float32")
+
+            if list_col:
+                values = _encode_list_column(df[name], values)
+
+            new_df[name] = values
         return new_df
 
     @property


Test patch:
diff --git a/tests/unit/ops/test_normalize.py b/tests/unit/ops/test_normalize.py
--- a/tests/unit/ops/test_normalize.py
+++ b/tests/unit/ops/test_normalize.py
@@ -22,7 +22,8 @@
 import nvtabular as nvt
 import nvtabular.io
 from nvtabular import ColumnSelector, dispatch, ops
-from nvtabular.dispatch import HAS_GPU
+from nvtabular.dispatch import HAS_GPU, _flatten_list_column, _flatten_list_column_values
+from tests.conftest import assert_eq
 
 if HAS_GPU:
     _CPU = [True, False]
@@ -83,6 +84,29 @@ def test_normalize(tmpdir, df, dataset, gpu_memory_frac, engine, op_columns):
         assert np.allclose(cupy_outputs[col], new_gdf[col].values)
 
 
+@pytest.mark.parametrize("cpu", _CPU)
+def test_normalize_lists(tmpdir, cpu):
+    df = dispatch._make_df(device="cpu" if cpu else "gpu")
+    df["vals"] = [
+        [0.0, 1.0, 2.0],
+        [
+            3.0,
+            4.0,
+        ],
+        [5.0],
+    ]
+
+    features = ["vals"] >> nvt.ops.Normalize()
+    workflow = nvt.Workflow(features)
+    transformed = workflow.fit_transform(nvt.Dataset(df)).to_ddf().compute()
+
+    expected = _flatten_list_column_values(df["vals"]).astype("float32")
+    expected = (expected - expected.mean()) / expected.std()
+    expected_df = type(transformed)({"vals": expected})
+
+    assert_eq(expected_df, _flatten_list_column(transformed["vals"]))
+
+
 @pytest.mark.parametrize("cpu", _CPU)
 def test_normalize_std_zero(cpu):
     df = pd.DataFrame({"a": 7 * [10]})