Repository: pegasystems/pega-datascientist-tools

Issue:
ADMDatamart init fails when data is empty and extracting keys
### pdstools version checks

- [X] I have checked that this issue has not already been reported.

- [X] I have confirmed this bug exists on the [latest version](https://pypi.org/project/pdstools/) of pdstools.


### Issue description

When passing an empty dataframe to ADMDatamart and setting extract_keys to True, the initialiser fails with an ugly polars exception.

Passing an empty dataframe can happen when filtering for certain date ranges like in some of the dashboarding applications. Extracting the treatments is standard and should probably default to True.

### Reproducible example

```Python
df=pdc.pdc_data.filter(pl.col("ModelType") == "AdaptiveModel").filter(current_reporting_period)
adm = ADMDatamart(model_df=df, extract_keys=False)
adm.is_available

The dataframe df is empty but has a schema:

shape: (0, 20)
ModelClass	ModelID	ModelName	ModelType	Name	Negatives	Performance	Positives	ResponseCount	SnapshotTime	Channel	Direction	Group	Issue	TotalPositives	ConfigurationName	TotalPredictors	ActivePredictors	CTR	ElapsedDays
str	str	str	str	str	f32	f32	f32	f32	date	str	str	str	str	str	str	i32	i32	f32	i64

==> False

So far so good, but now

adm = ADMDatamart(model_df=df, extract_keys=True)

Errors out with

SchemaError: invalid series dtype: expected `Struct`, got `null`
```


### Expected behavior

Expect to just continue and return False if asked for is_available.


### Installed versions

<details>

```
Replace this line with the output of pdstools.show_versions(), leave the backticks in place
```

</details>

Discussion:
@StijnKas: Right I can see how this would work. We call 'struct.field', if the column is not a struct at all, do you think we should just leave it as null? It doesn't look like it has anything to do with is_available@operdeck: I would expect the same return value (empty dataframe with a schema) regardless the value of "extract_keys". But if that costs some code complexity we can also just return None. The caller can easily check.@StijnKas: Tiny bit of complexity added - I'm simply doing a little pre-check to pull the very first row of the name column, and checking if the length of the df is >0. If not, I just return the original dataframe. Since extracting keys is eager anyway, this seems reasonable.

Solution patch:
diff --git a/python/pdstools/adm/BinAggregator.py b/python/pdstools/adm/BinAggregator.py
index aac3af72..02e86622 100644
--- a/python/pdstools/adm/BinAggregator.py
+++ b/python/pdstools/adm/BinAggregator.py
@@ -4,8 +4,8 @@ import plotly.express as px
 from plotly.graph_objects import Figure
 from typing import Union, Optional, Literal
 
-from pdstools import ADMDatamart
-from pdstools.utils.cdh_utils import lift  # only temp needed
+from .. import ADMDatamart
+from ..utils.cdh_utils import lift  # only temp needed
 
 # from IPython.display import display # for better display in notebooks rather than print of dataframes
 
@@ -305,9 +305,7 @@ class BinAggregator:
             values=["Lift"],
             index=["ModelID", "Lift_RESIDUAL"],
             columns="Symbol",
-        ).sort(
-            "ModelID"
-        )  # just for debugging/transparency
+        ).sort("ModelID")  # just for debugging/transparency
 
         if verbose:
             print("Pivot table:")
@@ -899,9 +897,7 @@ class BinAggregator:
             topic = binning.columns[-1]  # assuming the roll-up column is the last one
             model_facet = (
                 # if there are multiple topics, take the topic otherwise assume not rolled up over a topic
-                topic
-                if binning.select(pl.col(topic).n_unique() > 1).item()
-                else None
+                topic if binning.select(pl.col(topic).n_unique() > 1).item() else None
             )
             predictor_facet = (
                 # check if there are multiple predictors
diff --git a/python/pdstools/adm/Tables.py b/python/pdstools/adm/Tables.py
index a59cfde2..667409fb 100644
--- a/python/pdstools/adm/Tables.py
+++ b/python/pdstools/adm/Tables.py
@@ -1,5 +1,5 @@
 import polars as pl
-from pdstools.utils.cdh_utils import weighted_average_polars
+from ..utils.cdh_utils import weighted_average_polars
 from functools import cached_property
 
 # TODO: reconsier this whole class
diff --git a/python/pdstools/plots/plots_plotly.py b/python/pdstools/plots/plots_plotly.py
index c5665144..b2cc3fe3 100644
--- a/python/pdstools/plots/plots_plotly.py
+++ b/python/pdstools/plots/plots_plotly.py
@@ -9,7 +9,7 @@ import polars as pl
 import math
 from plotly.subplots import make_subplots
 from typing import Optional
-from pdstools.utils import pega_template
+from ..utils import pega_template
 
 
 class ADMVisualisations:
@@ -491,8 +491,8 @@ class ADMVisualisations:
         import plotly
         from packaging import version
 
-        assert version.parse(plotly.__version__) >= version.parse(
-            "5.5.0"
+        assert (
+            version.parse(plotly.__version__) >= version.parse("5.5.0")
         ), f"Visualisation requires plotly version 5.5.0 or later (you have version {plotly.__version__}): please upgrade to a newer version."
 
         title = "over all models" if facet is None else f"per {facet}"
diff --git a/python/pdstools/utils/cdh_utils.py b/python/pdstools/utils/cdh_utils.py
index 8fb8e898..a22ee70b 100644
--- a/python/pdstools/utils/cdh_utils.py
+++ b/python/pdstools/utils/cdh_utils.py
@@ -72,9 +72,17 @@ def _extract_keys(
     df: Union[pl.DataFrame, pl.LazyFrame]
         The dataframe to extract the keys from
     """
+    # Checking for the 'column is None/Null' case
+    if df.schema[col] != pl.Utf8:
+        return df
+
     if import_strategy != "eager":
         raise NotEagerError("Extracting keys")
 
+    # Checking for the 'empty df' case
+    if len(df.select(col).head(1).collect()) == 0:
+        return df
+
     def safeName():
         return (
             pl.when(~pl.col(col).cast(pl.Utf8).str.starts_with("{"))
@@ -604,6 +612,7 @@ def weighted_performance_polars() -> pl.Expr:
     """Polars function to return a weighted performance"""
     return weighted_average_polars("Performance", "ResponseCount")
 
+
 def overlap_lists_polars(col: pl.Series, row_validity: pl.Series) -> List[float]:
     """Calculate the overlap of each of the elements (must be a list) with all the others"""
     nrows = col.len()
@@ -618,14 +627,17 @@ def overlap_lists_polars(col: pl.Series, row_validity: pl.Series) -> List[float]
                 set_j = set(col[j].to_list())
                 intersection = set_i & set_j
                 overlap_w_other_channels += [len(intersection)]
-        if len(overlap_w_other_channels)>0:
+        if len(overlap_w_other_channels) > 0:
             average_overlap += [
-                sum(overlap_w_other_channels) / len(overlap_w_other_channels) / len(set_i)
+                sum(overlap_w_other_channels)
+                / len(overlap_w_other_channels)
+                / len(set_i)
             ]
         else:
             average_overlap += [float("nan")]
     return average_overlap
 
+
 def zRatio(
     posCol: pl.Expr = pl.col("BinPositives"), negCol: pl.Expr = pl.col("BinNegatives")
 ) -> pl.Expr:
diff --git a/python/pdstools/utils/hds_utils.py b/python/pdstools/utils/hds_utils.py
index facf22f5..ff27842d 100644
--- a/python/pdstools/utils/hds_utils.py
+++ b/python/pdstools/utils/hds_utils.py
@@ -7,7 +7,7 @@ import polars as pl
 
 from random import randint
 from typing import Literal, Optional, Union
-from pdstools import ADMDatamart
+from .. import ADMDatamart
 
 
 class Config:


Test patch:
diff --git a/python/tests/test_ADMDatamart.py b/python/tests/test_ADMDatamart.py
index 65961834..40ca4756 100644
--- a/python/tests/test_ADMDatamart.py
+++ b/python/tests/test_ADMDatamart.py
@@ -511,8 +511,8 @@ def test_overall_summary_cols(test):
         "Performance",
         "Positives",
         "ResponseCount",
-        'Minimum Performance',
-        'Channel with Minimum Performance',
+        "Minimum Performance",
+        "Channel with Minimum Performance",
         "Issues",
         "Groups",
         "Total Number of Actions",
@@ -526,8 +526,16 @@ def test_overall_summary_cols(test):
     ]
     assert len(summary) == 1
 
+
 def test_overall_summary_n_valid_channels(test):
     assert test.overall_summary().collect()["Number of Valid Channels"].item() == 3
 
+
 def test_overall_summary_positives(test):
     assert test.overall_summary().collect()["Positives"].item() == 106720
+
+
+def test_extract_keys_on_empty_df(test):
+    """Testing that an empty dataframe doesn't fail hard"""
+    filtered_df = test.modelData.filter(pl.col("Name") == "TEST")
+    ADMDatamart(model_df=filtered_df, extract_keys=True).modelData.collect()