Repository: mindsdb/lightwood

Issue:
[Bug] ICP softmax type error
## Your Environment
* Lightwood version: staging (23.1.2.1+)

## Describe your issue
`torch.softmax` is erroring due to underlying `str_` type in the tensor (which is quite weird):


```python
 File "/MindsDB/lightwood/lightwood/helpers/log.py", line 30, in wrap
    result = f(predictor, *args, **kw)
  File "/var/folders/_0/b5mtgvvs71gdbp7ftlyjdz340000gp/T/ff0526104935eccd8528c8d87b0b9045ab325dc013a3b2c7167468801567233.py", line 443, in learn
    self.analyze_ensemble(enc_train_test)
  File "/MindsDB/lightwood/lightwood/helpers/log.py", line 30, in wrap
    result = f(predictor, *args, **kw)
  File "/var/folders/_0/b5mtgvvs71gdbp7ftlyjdz340000gp/T/ff0526104935eccd8528c8d87b0b9045ab325dc013a3b2c7167468801567233.py", line 393, in analyze_ensemble
    self.model_analysis, self.runtime_analyzer = model_analyzer(
  File "/MindsDB/lightwood/lightwood/analysis/analyze.py", line 88, in model_analyzer
    runtime_analyzer = block.analyze(runtime_analyzer, **kwargs)
  File "/MindsDB/lightwood/lightwood/analysis/nc/calibrate.py", line 210, in analyze
    icps[tuple(group)].calibrate(icp_df.values, y)
  File "/MindsDB/lightwood/lightwood/analysis/nc/icp.py", line 102, in calibrate
    cal_scores = self.nc_function.score(self.cal_x, self.cal_y)
  File "/MindsDB/lightwood/lightwood/analysis/nc/nc.py", line 407, in score
    prediction = self.model.predict(x)
  File "/MindsDB/lightwood/lightwood/analysis/nc/base.py", line 165, in predict
    return t_softmax(self.prediction_cache, t=0.5)
  File "/MindsDB/lightwood/lightwood/analysis/nc/util.py", line 13, in t_softmax
    return softmax(torch.Tensor(x) / t, dim=axis).numpy()
TypeError: can't convert np.ndarray of type numpy.str_. The only supported types are: float64, float32, float16, complex64, complex128, int64, int32, int16, int8, uint8, and bool.
```

## How can we replicate it?
* What dataset did you use
[TS_R1S1_TRAIN_DATA.csv](https://github.com/mindsdb/lightwood/files/10504508/TS_R1S1_TRAIN_DATA.csv)

* What was the code you ran
Through MindsDB:

```sql
CREATE MODEL  
    mindsdb.TS_R1S1_SELL
FROM files  
    (SELECT SEQ, S1, CLOSE, SELL FROM files.TS_R1S1_TRAIN_DATA)
PREDICT SELL
ORDER BY SEQ 
GROUP BY S1
WINDOW 4
HORIZON 2 
```



Discussion:
As a workaround, I would like to confirm that the error can be bypassed by removing the "GROUP BY" clause and using the "use_default_analysis=False" in the query. (Thanks for @paxcema's advice)

```SQL
CREATE MODEL  
    mindsdb.TS_R1S1_SELL
FROM files  
    (SELECT SEQ, S1, CLOSE, SELL FROM files.TS_R1S1_TRAIN_DATA)
PREDICT SELL
ORDER BY SEQ 
-- GROUP BY S1
WINDOW 4
HORIZON 2 
USING
use_default_analysis=False;
```

I'm enclosing datasets using date/time instead of numbers for the SEQ column for the ordering.
[TS_R1S1_TRAIN_DATA_DT.csv](https://github.com/mindsdb/lightwood/files/10506236/TS_R1S1_TRAIN_DATA_DT.csv)
[TS_R1S1_TEST_DATA_DT.csv](https://github.com/mindsdb/lightwood/files/10506237/TS_R1S1_TEST_DATA_DT.csv)


Resolving patch:
diff --git a/lightwood/analysis/nc/calibrate.py b/lightwood/analysis/nc/calibrate.py
--- a/lightwood/analysis/nc/calibrate.py
+++ b/lightwood/analysis/nc/calibrate.py
@@ -105,19 +105,20 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
             icp_df = deepcopy(ns.data)
 
             # setup prediction cache to avoid additional .predict() calls
-            pred_is_list = isinstance(ns.normal_predictions['prediction'], list) and \
-                isinstance(ns.normal_predictions['prediction'][0], list)
+            try:
+                pred_is_list = isinstance(ns.normal_predictions['prediction'][0], list)
+            except KeyError:
+                pred_is_list = False
+
             if ns.is_classification:
                 if ns.predictor.supports_proba:
                     icp.nc_function.model.prediction_cache = ns.normal_predictions[all_cat_cols].values
                 else:
                     if ns.is_multi_ts:
-                        icp.nc_function.model.prediction_cache = np.array(
-                            [p[0] for p in ns.normal_predictions['prediction']])
-                        preds = icp.nc_function.model.prediction_cache
+                        preds = np.array([p[0] for p in ns.normal_predictions['prediction']])
                     else:
-                        preds = ns.normal_predictions['prediction']
-                    predicted_classes = pd.get_dummies(preds).values  # inflate to one-hot enc
+                        preds = ns.normal_predictions['prediction'].values
+                    predicted_classes = output['label_encoders'].transform(preds.reshape(-1, 1))  # inflate OHE
                     icp.nc_function.model.prediction_cache = predicted_classes
 
             elif ns.is_multi_ts or pred_is_list:
@@ -198,8 +199,12 @@ def analyze(self, info: Dict[str, object], **kwargs) -> Dict[str, object]:
 
                     # save relevant predictions in the caches, then calibrate the ICP
                     pred_cache = icp_df.pop(f'__predicted_{ns.target}').values
-                    if ns.is_multi_ts:
+                    if ns.is_multi_ts and ns.is_classification:
+                        # output['label_encoders'].transform(preds.reshape(-1, 1))
+                        pred_cache = output['label_encoders'].transform([[p[0] for p in pred_cache]])
+                    elif ns.is_multi_ts:
                         pred_cache = np.array([np.array(p) for p in pred_cache])
+
                     icps[tuple(group)].nc_function.model.prediction_cache = pred_cache
                     icp_df, y = clean_df(icp_df, ns, output.get('label_encoders', None))
                     if icps[tuple(group)].nc_function.normalizer is not None:
@@ -334,7 +339,8 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
                         for icol, cat_col in enumerate(all_cat_cols):
                             row_insights.loc[X.index, cat_col] = class_dists[:, icol]
                     else:
-                        class_dists = pd.get_dummies(preds).values
+                        ohe_enc = ns.analysis['label_encoders']
+                        class_dists = ohe_enc.transform(np.array([p[0] for p in preds]).reshape(-1, 1))
 
                     base_icp.nc_function.model.prediction_cache = class_dists
 
@@ -360,8 +366,8 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
                     result.loc[X.index, 'significance'] = significances
 
                 else:
-                    significances = get_categorical_conf(all_confs.squeeze())
-                    result.loc[X.index, 'significance'] = significances
+                    significances = get_categorical_conf(all_confs)
+                    result.loc[X.index, 'significance'] = significances.flatten()
 
                 # grouped time series, we replace bounds in rows that have a trained ICP
                 if ns.analysis['icp'].get('__mdb_groups', False):
@@ -386,6 +392,11 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
                                                                       for i in range(1, ns.tss.horizon)]
                                     icp.nc_function.model.prediction_cache = X[target_cols].values
                                     [X.pop(col) for col in target_cols]
+                                elif is_multi_ts and is_categorical:
+                                    ohe_enc = ns.analysis['label_encoders']
+                                    preds = X.pop(ns.target_name).values
+                                    pred_cache = ohe_enc.transform(np.array([p[0] for p in preds]).reshape(-1, 1))
+                                    icp.nc_function.model.prediction_cache = pred_cache
                                 else:
                                     icp.nc_function.model.prediction_cache = X.pop(ns.target_name).values
                                 if icp.nc_function.normalizer:
@@ -431,7 +442,7 @@ def explain(self, row_insights: pd.DataFrame, global_insights: Dict[str, object]
                                     all_ranges = np.array([icp.predict(X.values)])
                                     all_confs = np.swapaxes(np.swapaxes(all_ranges, 0, 2), 0, 1)
                                     significances = get_categorical_conf(all_confs)
-                                    result.loc[X.index, 'significance'] = significances
+                                    result.loc[X.index, 'significance'] = significances.flatten()
 
                 row_insights['confidence'] = result['significance']
 
@@ -519,6 +530,5 @@ def _ts_assign_confs(result, df, confs, significances, tss) -> pd.DataFrame:
             added_cols = [f'{base_col}_timestep_{t}' for t in range(1, tss.horizon)]
             cols = [base_col] + added_cols
             result.loc[df.index, base_col] = result.loc[df.index, cols].values.tolist()
-            # result[base_col] = result[cols].values.tolist()
 
         return result
diff --git a/lightwood/data/timeseries_transform.py b/lightwood/data/timeseries_transform.py
--- a/lightwood/data/timeseries_transform.py
+++ b/lightwood/data/timeseries_transform.py
@@ -69,7 +69,7 @@ def transform_timeseries(
     subsets = []
     for group in groups:
         if (tss.group_by and group != '__default') or not tss.group_by:
-            idxs, subset = get_group_matches(data, group, tss.group_by)
+            idxs, subset = get_group_matches(data, group, tss.group_by, copy=True)
             if subset.shape[0] > 0:
                 if periods.get(group, periods['__default']) == 0 and subset.shape[0] > 1:
                     raise Exception(
diff --git a/lightwood/helpers/ts.py b/lightwood/helpers/ts.py
--- a/lightwood/helpers/ts.py
+++ b/lightwood/helpers/ts.py
@@ -18,7 +18,8 @@ def get_ts_groups(df: pd.DataFrame, tss) -> list:
 def get_group_matches(
         data: Union[pd.Series, pd.DataFrame],
         combination: tuple,
-        group_columns: List[str]
+        group_columns: List[str],
+        copy: bool = False
 ) -> Tuple[list, pd.DataFrame]:
     """Given a particular group combination, return the data subset that belongs to it."""
 
@@ -34,6 +35,8 @@ def get_group_matches(
         for val, col in zip(combination, group_columns):
             subset = subset[subset[col] == val]
         if len(subset) > 0:
+            if copy:
+                subset = subset.copy()
             return list(subset.index), subset
         else:
             return [], pd.DataFrame()


Test patch: