Repository: evidentlyai/evidently
Issue: Reports on Spark not providing output
Root cause: Validation Gap

Hello,

I am using SparkEngine to generate `DataDriftPreset` Report. When I use example datasets, everything works fine. The code:
```
    def spark_data_generation(n_rows=3000, n_columns=5):
        return spark_session.range(n_rows).select(
            *[sf.rand().alias(str(i)) for i in range(n_columns)]
        )

    reference = spark_data_generation()
    current = spark_data_generation()

    report = Report(
        metrics=[DataDriftPreset(stattest="jensenshannon", stattest_threshold=0.05)]
    )

    report.run(
        reference_data=reference,
        current_data=current,
        engine=SparkEngine,
    )

    report.save("snapshot.json")
    with open("snapshot.json") as f:
        loaded_report = json.dumps(json.load(f))
```
Both `report.json()` and `loaded_report` (saved `snapshot.json` file) work fine and contain expected metrics and statistics.

But when switching to my datasets, `report.json()` outputs 
```
'  File ' |  
-- | --
  |   | '"/yarn/disk1/local-dirs/usercache/data-science/appcache/application_1734990993705_123114/container_e66_1734990993705_123114_01_000001/environment/lib/python3.12/site-packages/evidently/spark/calculations/histogram.py", ' |  
  |   | 'line 70, in value_counts\n' |  
  |   | '    return {r[column_name]: r["count"] for r in ' |  
  |   | 'data.groupby(column_name).count().collect()}\n' |  
  |   | '            ^^^^^^^^^^^^^^^^^^^^^^^^^^\n' |  
  |   | "TypeError: unhashable type: 'list'\n") |  
  |   | unhashable type: 'list'.
```
and `loaded_report` outputs this:
```
{
  "id": "01948ec7-6abf-78a0-a5d1-a848f514d9c4",
  "name": null,
  "timestamp": "2025-01-22T16:10:21.248006",
  "metadata": {},
  "tags": [],
  "suite": {
    "metrics": [
      {
        "type": "evidently:metric:DatasetDriftMetric",
        "options": {
          "color": null,
          "render": null,
          "data_definition": null
        },
        "stattest": null,
        "cat_stattest": null,
        "num_stattest": null,
        "text_stattest": null,
        "per_column_stattest": null,
        "stattest_threshold": null,
        "cat_stattest_threshold": null,
        "num_stattest_threshold": null,
        "text_stattest_threshold": null,
        "per_column_stattest_threshold": null,
        "columns": null,
        "drift_share": 0.5
      },
      {
        "type": "evidently:metric:DataDriftTable",
        "options": {
          "color": null,
          "render": null,
          "data_definition": null
        },
        "stattest": null,
        "cat_stattest": null,
        "num_stattest": null,
        "text_stattest": null,
        "per_column_stattest": null,
        "stattest_threshold": null,
        "cat_stattest_threshold": null,
        "num_stattest_threshold": null,
        "text_stattest_threshold": null,
        "per_column_stattest_threshold": null,
        "columns": null,
        "feature_importance": false
      }
    ],
    "metric_results": [
      {
        "type": "evidently:metric_result:MetricResult"
      },
      {
        "type": "evidently:metric_result:MetricResult"
      }
    ],
    "tests": [],
    "test_results": [],
    "options": {
      "color": null,
      "render": null,
      "data_definition": null
    },
    "data_definition": {
      "columns": {
        "duration": {
          "column_name": "duration",
          "column_type": "num"
        },
        "sellerId": {
          "column_name": "sellerId",
          "column_type": "num"
        },
        "vplay": {
          "column_name": "vplay",
          "column_type": "cat"
        },
        "byrNl": {
          "column_name": "byrNl",
          "column_type": "cat"
        },
        "country": {
          "column_name": "country",
          "column_type": "cat"
        },
        "browser": {
          "column_name": "browser",
          "column_type": "cat"
        },
        "videoPosition": {
          "column_name": "videoPosition",
          "column_type": "cat"
        },
        "siteDomain": {
          "column_name": "siteDomain",
          "column_type": "cat"
        },
        "os": {
          "column_name": "os",
          "column_type": "cat"
        },
        "osversion": {
          "column_name": "osversion",
          "column_type": "cat"
        },
        "browserV": {
          "column_name": "browserV",
          "column_type": "cat"
        },
        "contUrl": {
          "column_name": "contUrl",
          "column_type": "cat"
        },
        "isp": {
          "column_name": "isp",
          "column_type": "cat"
        },
        "sappCat": {
          "column_name": "sappCat",
          "column_type": "cat"
        },
        "region": {
          "column_name": "region",
          "column_type": "cat"
        },
        "org": {
          "column_name": "org",
          "column_type": "cat"
        },
        "sappName": {
          "column_name": "sappName",
          "column_type": "cat"
        },
        "pageL": {
          "column_name": "pageL",
          "column_type": "cat"
        },
        "creativeSize": {
          "column_name": "creativeSize",
          "column_type": "cat"
        },
        "vplace": {
          "column_name": "vplace",
          "column_type": "cat"
        },
        "pubId": {
          "column_name": "pubId",
          "column_type": "cat"
        },
        "bannerPosition": {
          "column_name": "bannerPosition",
          "column_type": "cat"
        },
        "deviceType": {
          "column_name": "deviceType",
          "column_type": "cat"
        },
        "uidNl": {
          "column_name": "uidNl",
          "column_type": "cat"
        },
        "floor": {
          "column_name": "floor",
          "column_type": "cat"
        }
      },
      "target": null,
      "prediction_columns": null,
      "id_column": null,
      "datetime_column": null,
      "embeddings": null,
      "user_id": {
        "column_name": "user_id",
        "column_type": "id"
      },
      "item_id": {
        "column_name": "item_id",
        "column_type": "id"
      },
      "task": null,
      "classification_labels": null,
      "reference_present": true,
      "recommendations_type": "score"
    },
    "run_metadata": {
      "descriptors": {}
    }
  },
  "metrics_ids": [
    0,
    1
  ],
  "test_ids": [],
  "options": {
    "color": null,
    "render": null,
    "data_definition": null
  },
  "links": {
    "datasets": {
      "input": {
        "reference": null,
        "current": null,
        "additional": {}
      },
      "output": {
        "reference": null,
        "current": null,
        "additional": {}
      }
    },
    "computation_config_id": null,
    "task_id": null
  }
}
```
It is missing actual metrics and statistics. Does anyone know what the problem could be?

Additionally, pandas implementation on my datasets works fine

I think I found the answer. In my case, I need to define all column types for `ColumnMapping()` object. Also, some of the categorical features are making job output empry `json` file as I provided above, so removing them made this work.