update

app.py

@@ -1,6 +1,4 @@
 import streamlit as st
-from flaml.automl.data import get_output_from_log
-import plotly.express as px
 
 from utils import csv_to_featuers_list
 from autoML import autoML

autoML.py

@@ -7,8 +7,11 @@ import pickle
 import plotly.express as px
 import base64
 import time
-
-from utils import csv_to_featuers_list, pre_process_df, pre_process_features
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.impute import SimpleImputer
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.inspection import permutation_importance
 
 def autoML(csv, task, budget, label, metric_to_minimize_class, metric_to_minimize_reg):
 
@@ -17,9 +20,7 @@ def autoML(csv, task, budget, label, metric_to_minimize_class, metric_to_minimiz
     time.sleep(0.5)
 
     df = pd.read_csv(csv)
-    df = pre_process_df(df)
     df_features = df[df.columns.difference([label])]
-    df_features=(df_features-df_features.mean())/df_features.std()
     y = df[label]
 
     my_bar.progress(50, text=progress_text)
@@ -49,8 +50,31 @@ def autoML(csv, task, budget, label, metric_to_minimize_class, metric_to_minimiz
             "eval_method": "holdout"
         }
 
-    automl = AutoML()
-    automl.fit(df_features, y, **automl_settings)
+    num_cols = df_features.select_dtypes(include=['float64', 'int64']).columns
+    cat_cols = df_features.select_dtypes(include=['object']).columns
+
+    numeric_transformer = Pipeline(steps=[
+        ('imputer', SimpleImputer(strategy='mean')),
+        ('scaler', StandardScaler())
+    ])
+
+    categorical_transformer = Pipeline(steps=[
+        ('imputer', SimpleImputer(strategy='most_frequent')),
+        ('onehot', OneHotEncoder(handle_unknown='ignore'))
+    ])
+
+    preprocessor = ColumnTransformer(
+        transformers=[
+            ('num', numeric_transformer, num_cols),
+            ('cat', categorical_transformer, cat_cols)
+        ])
+
+    automl = AutoML(**automl_settings)
+
+    pipeline = Pipeline(steps=[('preprocessor', preprocessor),
+                                ('classifier', automl)])
+
+    pipeline.fit(df_features, y)
 
     my_bar.progress(100, text=progress_text)
     time.sleep(0.5)
@@ -112,13 +136,18 @@ def autoML(csv, task, budget, label, metric_to_minimize_class, metric_to_minimiz
         with col3:
             st.metric(label="Time to train", value=str(round(automl.best_config_train_time, 2))+' sec')
 
-        if automl.best_estimator == 'lgbm':
-            df_features_importance = pd.DataFrame({'features name': automl.model.estimator.feature_name_, 'features importance': automl.model.estimator.feature_importances_})
-            fig_features = px.bar(df_features_importance, x='features importance', y='features name')
-
-            st.divider()
-            st.plotly_chart(fig_features, theme="streamlit")
+        perm_importance = permutation_importance(
+            pipeline, df_features, y, n_repeats=8
+        )
 
+        df_features_importance = pd.DataFrame({'features name': df_features.columns, 
+                                                'features importance': perm_importance["importances_mean"],
+                                                'std error': perm_importance["importances_std"]})
+        
+        fig_features = px.bar(df_features_importance, x='features importance', y='features name', error_x='std error')
+    
+        st.divider()
+        st.plotly_chart(fig_features, theme="streamlit")
 
         def download_model(model):
             output_model = pickle.dumps(model)