Upload 3 files

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 import pandas as pd
 
-from data_generation import generate_synthetic_training_data
+from data_generation import generate_synthetic_training_data, evaluate_synthetic_data
 from data_preprocessing import preprocess_real_data, preprocess_synthetic_data
 from credit_models import real_data_credit_model, synthetic_data_credit_model
 from visualization import (
@@ -10,6 +10,7 @@ from visualization import (
     plot_comparison_table,
     plot_comparative_confusion_matrices,
     plot_comparative_credit_score_distribution_by_actual_class,
+    plot_evaluation_table,
     get_metrics_df,
 )
 
@@ -23,8 +24,8 @@ LABEL_ORDER = ['Good', 'Standard', 'Poor']
 TARGET = 'Credit_Score'
 
 # Load and preprocess real data once at startup
-real_train = pd.read_csv('data/processed/v4/real_train_data.csv')
-real_test = pd.read_csv('data/processed/v4/real_test_data.csv')
+real_train = pd.read_csv('../data/processed/v4/real_train_data.csv')
+real_test = pd.read_csv('../data/processed/v4/real_test_data.csv')
 
 X_real_train, y_real_train, X_real_test, y_real_test = preprocess_real_data(
     real_train, real_test, TARGET
@@ -35,16 +36,34 @@ real_scores, real_classification = real_data_credit_model(
     X_real_train, y_real_train, X_real_test
 )
 
-
 def run_analysis():
     """Generate new synthetic data, train the synthetic model, and return all comparison plots."""
     synthetic_data = generate_synthetic_training_data(n=int(len(X_real_train)/3)) # Same number of samples as real training data
     X_synth_train, y_synth_train = preprocess_synthetic_data(synthetic_data, TARGET)
 
+    categorical_cols = [col for col in X_real_train.columns if X_real_train[col].dtype in ["object", "bool", "uint8"]]
+    numeric_cols = [col for col in X_real_train.columns if X_real_train[col].dtype in ["int64", "float64"]]
+
     fig_feature_dist = plot_feature_distributions(
         X_real_train, X_synth_train
     )
 
+    summary_rows = []
+    for cls in LABEL_ORDER:
+        real_cls = X_real_train[y_real_train == cls]
+        synth_cls = X_synth_train[y_synth_train == cls]
+        ks_pass_rate, mean_ks, chi_pass_rate, mean_corr_diff = evaluate_synthetic_data(
+            real_cls, synth_cls, categorical_cols, numeric_cols
+        )
+        summary_rows.append({
+            "ks_passed": ks_pass_rate == 1.0,
+            "mean_ks_stat": round(mean_ks, 4),
+            "chi_passed": chi_pass_rate == 1.0 if chi_pass_rate is not None else None,
+            "mean_corr_diff": round(mean_corr_diff, 4),
+        })
+    summary_df = pd.DataFrame(summary_rows, index=LABEL_ORDER)
+    summary = plot_evaluation_table(summary_df)
+
     synth_scores, synth_classification = synthetic_data_credit_model(
         X_synth_train, y_synth_train, X_real_test
     )
@@ -68,7 +87,7 @@ def run_analysis():
     metrics_df = get_metrics_df(y_real_test, real_classification, synth_classification)
     metrics_df = metrics_df.round(4)
 
-    return fig_feature_dist, fig_score_dist, fig_score_by_class, fig_metrics, fig_cm, metrics_df
+    return fig_feature_dist, summary, fig_score_dist, fig_score_by_class, fig_metrics, fig_cm, metrics_df
 
 
 with gr.Blocks(title="Credit Score Model Dashboard", theme=gr.themes.Soft()) as demo:
@@ -95,6 +114,16 @@ with gr.Blocks(title="Credit Score Model Dashboard", theme=gr.themes.Soft()) as
     with gr.Row():
         plot_feature_dist = gr.Plot(label='')
 
+    gr.Markdown(
+    """
+    ## Generated Data Quality Summary
+
+    Below is a summary of the data quality evaluation comparing the synthetic training data to the real training data across multiple metrics.
+    """
+    )
+
+    with gr.Row():
+        plot_summary = gr.Plot(label='')
 
     gr.Markdown(
     """
@@ -140,7 +169,7 @@ with gr.Blocks(title="Credit Score Model Dashboard", theme=gr.themes.Soft()) as
     run_btn.click(
         fn=run_analysis,
         inputs=[],
-        outputs=[plot_feature_dist, plot_score_dist, plot_score_by_class, plot_metrics, plot_cm],
+        outputs=[plot_feature_dist, plot_summary, plot_score_dist, plot_score_by_class, plot_metrics, plot_cm],
     )
 
 demo.launch()

data_generation.py

@@ -1,6 +1,8 @@
 import torch
 import pandas as pd
 from sdv.single_table import CTGANSynthesizer
+from scipy import stats
+import numpy as np
 
 # Patch torch.load to remap MPS tensors to CPU for environments without Apple Silicon
 _original_torch_load = torch.load
@@ -20,9 +22,9 @@ def generate_synthetic_training_data(n=30_000):
     Returns:
         pd.DataFrame: The generated synthetic training data.
     """
-    good_generator = CTGANSynthesizer.load("models/v4/synth_good.pkl")
-    poor_generator = CTGANSynthesizer.load("models/v4/synth_poor.pkl")
-    standard_generator = CTGANSynthesizer.load("models/v4/synth_standard.pkl")
+    good_generator = CTGANSynthesizer.load("../models/v4/synth_good.pkl")
+    poor_generator = CTGANSynthesizer.load("../models/v4/synth_poor.pkl")
+    standard_generator = CTGANSynthesizer.load("../models/v4/synth_standard.pkl")
 
     synth_good = good_generator.sample(n)
     synth_poor = poor_generator.sample(n)
@@ -30,4 +32,88 @@ def generate_synthetic_training_data(n=30_000):
 
     full_data = pd.concat([synth_good, synth_poor, synth_standard], ignore_index=True)
     shuffled_data = full_data.sample(frac=1).reset_index(drop=True)
-    return shuffled_data
+    return shuffled_data
+
+
+def evaluate_synthetic_data(real_df, synthetic_df,
+                             categorical_cols=None, numeric_cols=None):
+    if numeric_cols is None:
+        numeric_cols = real_df.select_dtypes(include="number").columns.tolist()
+    if categorical_cols is None:
+        categorical_cols = real_df.select_dtypes(include="object").columns.tolist()
+
+    # ── KS Test ───────────────────────────────────────────────────────────
+    ks_results = []
+    for col in numeric_cols:
+        stat, p_value = stats.ks_2samp(
+            real_df[col].dropna(),
+            synthetic_df[col].dropna()
+        )
+        ks_results.append({
+            "column" : col,
+            "ks_stat": round(stat, 4),
+            "p_value": round(p_value, 4),
+            "pass"   : p_value > 0.05
+        })
+    ks_df = pd.DataFrame(ks_results)
+
+    # ── Chi-Square Test ───────────────────────────────────────────────────
+    chi_results = []
+    for col in categorical_cols:
+        real_counts  = real_df[col].value_counts()
+        synth_counts = synthetic_df[col].value_counts()
+        all_cats     = real_counts.index.union(synth_counts.index)
+        real_freq    = real_counts.reindex(all_cats, fill_value=0)
+        synth_freq   = synth_counts.reindex(all_cats, fill_value=0)
+        n            = real_freq.sum()
+        f_exp        = (real_freq  / real_freq.sum())  * n
+        f_obs        = (synth_freq / synth_freq.sum()) * n
+        stat, p_value = stats.chisquare(f_obs=f_obs, f_exp=f_exp)
+        chi_results.append({
+            "column"  : col,
+            "chi_stat": round(stat, 4),
+            "p_value" : round(p_value, 4),
+            "pass"    : p_value > 0.05
+        })
+    chi_df = pd.DataFrame(chi_results)
+
+    # ── Correlation Matrix ────────────────────────────────────────────────
+    real_corr      = real_df[numeric_cols].corr()
+    synth_corr     = synthetic_df[numeric_cols].corr()
+    corr_diff      = (real_corr - synth_corr).abs()
+    upper_idx      = np.triu_indices_from(corr_diff.values, k=1)
+    mean_corr_diff = corr_diff.values[upper_idx].mean()
+
+    ks_pass_rate  = ks_df["pass"].mean()
+    mean_ks       = ks_df["ks_stat"].mean()
+    chi_pass_rate = chi_df["pass"].mean() if not chi_df.empty else None
+
+    return ks_pass_rate, mean_ks, chi_pass_rate, mean_corr_diff
+
+
+def data_evaluation(
+        real_list, synthetic_list, class_names=None, categorical_cols=None, numeric_cols=None
+):
+    if class_names is None:
+        class_names = ["good", "poor", "standard"]
+
+    summary_rows = []
+
+    for cls, real_df, syn_df in zip(class_names, real_list, synthetic_list):
+
+        ks_pass_rate, mean_ks, chi_pass_rate, mean_corr_diff = evaluate_synthetic_data(
+            real_df          = real_df,
+            synthetic_df     = syn_df,
+            categorical_cols = categorical_cols,
+            numeric_cols     = numeric_cols
+        )
+
+        summary_rows.append({
+            "class"         : cls,
+            "ks_pass_rate"  : round(ks_pass_rate, 4),
+            "mean_ks_stat"  : round(mean_ks, 4),
+            "chi_pass_rate" : round(chi_pass_rate, 4) if chi_pass_rate is not None else None,
+            "mean_corr_diff": round(mean_corr_diff, 4)
+        })
+
+    return pd.DataFrame(summary_rows).set_index("class")

visualization.py

@@ -258,5 +258,49 @@ def plot_comparative_credit_score_distribution_by_actual_class(
     ax_right.set_ylabel('Frequency')
     ax_right.legend(title='Actual Class')
 
+    plt.tight_layout()
+    return fig
+
+
+def plot_evaluation_table(summary_df, title="Synthetic Data Evaluation Summary"):
+    display_df = summary_df.copy().round(4)
+
+    fig, ax = plt.subplots(figsize=(18, 2))
+    ax.axis("off")
+
+    table = ax.table(
+        cellText  = display_df.values,
+        rowLabels = display_df.index,
+        colLabels = display_df.columns,
+        cellLoc   = "center",
+        loc       = "center",
+    )
+    table.auto_set_font_size(False)
+    table.set_fontsize(16)
+    table.scale(1.2, 1.9)
+
+    for j in range(len(display_df.columns)):
+        table[(0, j)].set_facecolor("#1F77B4")
+        table[(0, j)].set_text_props(color="white", weight="bold")
+        table[(0, j)].set_edgecolor("white")
+        table[(0, j)].set_linewidth(1)
+
+    for i in range(1, len(display_df.index) + 1):
+        bg = "#0B0F19"
+
+        table[(i, -1)].set_text_props(color="white", weight="bold")
+        table[(i, -1)].set_facecolor(bg)
+        table[(i, -1)].set_edgecolor("white")
+        table[(i, -1)].set_linewidth(1)
+
+        for j in range(len(display_df.columns)):
+            table[(i, j)].set_facecolor(bg)
+            table[(i, j)].set_text_props(color="white")
+            table[(i, j)].set_edgecolor("white")
+            table[(i, j)].set_linewidth(1)
+
+    ax.set_title(title, color="white", fontsize=16, weight="bold", pad=12)
+    fig.patch.set_facecolor("#0B0F19")
+
     plt.tight_layout()
     return fig