fix shap plot

app.py

@@ -5,8 +5,14 @@ import pandas as pd
 import matplotlib.pyplot as plt
 import numpy as np
 import shap
+from sklearn.pipeline import make_pipeline
 
-project = hopsworks.login(project="SonyaStern_Lab1")
+feature_names = ["Age", "BMI", "HbA1c", "Blood Glucose"]
+
+project = hopsworks.login(
+    project="SonyaStern_Lab1",
+    api_key_value="c9StuuVQPoMUeXWe.jB2XeWcI8poKUN59W13MxAbMemzY7SChOnX151GtTFNhysBBUPMRuEp5IK7SE3i1",
+)
 fs = project.get_feature_store()
 
 print("trying to dl model")
@@ -51,11 +57,14 @@ with gr.Blocks() as demo:
             )
             btn = gr.Button("Submit")
         with gr.Column():
-            output = gr.Text(label="Model prediction")
-            plot = gr.Plot()
+            with gr.Row():
+                output = gr.Text(label="Model prediction")
+            with gr.Row():
+                mean_plot = gr.Plot()
     with gr.Row():
         with gr.Accordion("See model explanability", open=False):
-            waterfall_plot = gr.Plot()
+            with gr.Column():
+                waterfall_plot = gr.Plot()
 
     def submit_inputs(
         age_input,
@@ -105,23 +114,42 @@ with gr.Blocks() as demo:
         ax.legend()
         ax.set_xlabel("Variables")
         ax.set_ylabel("Values")
-        ax.set_title("Comparison with Mean values for your age")
+        ax.set_title("Comparison with average non-diabetic values for your age")
         ax.set_xticks(indices + bar_width / 2)
         ax.set_xticklabels(categories)
 
         ## explainability plots
-        rf_model = model.steps[-1][1]
+        rf_classifier = model.named_steps["randomforestclassifier"]
+        transformer_pipeline = make_pipeline(
+            *[
+                step
+                for name, step in model.named_steps.items()
+                if name != "randomforestclassifier"
+            ]
+        )
+        transformed_df = transformer_pipeline.transform(df)
+
+        # Generate the SHAP waterfall plot for fig2
+        explainer = shap.TreeExplainer(rf_classifier)
+        shap_values = explainer.shap_values(
+            transformed_df
+        )  # Compute SHAP values directly on the DataFrame
+        predicted_class = rf_classifier.predict(transformed_df)[0]
+        shap_values_for_predicted_class = shap_values[predicted_class]
 
-        explainer = shap.Explainer(rf_model)
-        shap_values = explainer.shap_values(df)[1]  # Select SHAP values for class 1
-        print(shap_values.shape)  # should show (1, 4)
-        # Convert shap_values to Explanation object
-        shap_values_exp = shap.Explanation(
-            values=shap_values[0], base_values=explainer.expected_value[1]
+        # Select the SHAP values for the first instance and the positive class
+        shap_explanation = shap.Explanation(
+            values=shap_values_for_predicted_class[0],
+            base_values=explainer.expected_value[predicted_class],
+            data=transformed_df[0],
+            feature_names=df.columns.tolist(),
         )
 
-        # Plot the waterfall plot
-        shap_waterfall_plot = shap.plots.waterfall(shap_values_exp)
+        fig2 = plt.figure(figsize=(12, 6))  # Create a new figure for SHAP plot
+        shap.waterfall_plot(
+            shap_explanation
+        )  # Set show=False to prevent immediate display
+        plt.title("SHAP Waterfall Plot")  # Optionally set a title for the SHAP plot
 
         ## save user's data in hopsworks
         if consent_input == True:
@@ -135,7 +163,7 @@ with gr.Blocks() as demo:
             user_data_df["diabetes"] = existent_info_input
             user_data_fg.insert(user_data_df)
             print("inserted new user data to hopsworks", user_data_df)
-        return res, fig, shap_waterfall_plot
+        return res, fig, fig2
 
     btn.click(
         submit_inputs,
@@ -147,7 +175,7 @@ with gr.Blocks() as demo:
             existent_info_input,
             consent_input,
         ],
-        outputs=[output, plot, waterfall_plot],
+        outputs=[output, mean_plot, waterfall_plot],
     )
 
 demo.launch()