SARIMA and XGBoost bugs fixed

app.py

@@ -6,10 +6,11 @@ import statsmodels.api as sm
 import matplotlib.pyplot as plt
 from utils.dataprocess import (load_data, df_col, numberOfDiff,create_features, 
                                apply_moving_average, apply_exponential_average,
-                               first_order_diff, second_order_diff, isStatinary)
+                               first_order_diff, second_order_diff, isStatinary,
+                               inverse_first_order_diff)
 from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
 from utils.graphics import plotDecompse, plotTs, plotForcast, plotTransformation
-from models.arima import SARIMAXGridSearch, valid_model, sarimax_forecast
+from models.arima import auto_arima, sarimax_forecast
 from utils.constants import DEFAULT_DATASETS_DIR, FREQ_DICT
 import xgboost as xgb
 from models.xgboost import xgboost
@@ -63,42 +64,46 @@ df_copy = df.copy()
 # Data Transformation
 # Sélection des transformations dans l'ordre
 sider.divider()
+
 transformations_function = [np.log,np.sqrt, first_order_diff, second_order_diff]
-transformations = [t.__name__ for t in transformations_function]
-selected_transformations = sider.multiselect("Data Transfomations", transformations)
-
-# Apply transformations
-
-df["Transformed"] = df["data"]
-transform_name = ""
-for transformation in selected_transformations:
-    if transformation == np.log.__name__:
-        try:
-            df["Transformed"] = np.log(df["Transformed"])
-            transform_name = transform_name + "log_"
-        except:
-            st.error(f"Unable to apply {np.log.__name__}")
-    elif transformation == np.sqrt.__name__:
-        try:
-            df["Transformed"] = np.sqrt(df["Transformed"])
-            transform_name = transform_name + "sqrt_"
-        except:
-            st.error(f"Unable to apply {np.sqrt.__name__}")
-            
-    elif transformation == "first_order_diff":
-        try:
-            df["Transformed"] = first_order_diff(df["Transformed"])
-            transform_name = transform_name + "diff1_"
-        except:
-            st.error(f"Unable to apply {transformation}")
-    elif transformation == "second_order_diff":
-        try:
-            df["Transformed"] = second_order_diff(df["Transformed"])
-            transform_name = transform_name + "diff2_"
-        except:
-            st.error(f"Unable to apply {transformation}")
-transform_name = transform_name[:-1]
+def transform_view(df, model="",transform_name="", key="transform"):
+    transf_col = "Transformed"+model
+    
+    transformations = [t.__name__ for t in transformations_function]
+    selected_transformations = sider.multiselect("Data Transfomations", transformations)
+
+    # Apply transformations
 
+    df[transf_col] = df["data"]
+    for transformation in selected_transformations:
+        if transformation == np.log.__name__:
+            try:
+                df[transf_col] = np.log(df[transf_col])
+                transform_name = transform_name + "log_"
+            except:
+                st.error(f"Unable to apply {np.log.__name__}")
+        elif transformation == np.sqrt.__name__:
+            try:
+                df[transf_col] = np.sqrt(df[transf_col])
+                transform_name = transform_name + "sqrt_"
+            except:
+                st.error(f"Unable to apply {np.sqrt.__name__}")
+                
+        elif transformation == "first_order_diff":
+            try:
+                df[transf_col], lags = first_order_diff(df[transf_col])
+                transform_name = transform_name + f"diff-{lags}_"
+            except:
+                st.error(f"Unable to apply {transformation}")
+        elif transformation == "second_order_diff":
+            try:
+                df[transf_col] = second_order_diff(df[transf_col])
+                transform_name = transform_name + "diff2_"
+            except:
+                st.error(f"Unable to apply {transformation}")
+    transform_name = transform_name[:-1]
+    return df,transform_name
+df, transform_name = transform_view(df)
 # Graphics
 st.subheader("Visualize your Time Series")
 
@@ -136,12 +141,10 @@ with pacf_acf:
         plot_acf(df["data"],  ax=ax1)
         plot_pacf(df["data"],  ax=ax2)
     else:
-        plot_acf(df[choice],  ax=ax1)
-        plot_pacf(df[choice],  ax=ax2)
+        plot_acf(df[choice].dropna(),  ax=ax1)
+        plot_pacf(df[choice].dropna(),  ax=ax2)
     st.pyplot(fig2)
-
-sider.divider()
-
+    
 # Split the df
 train_size = int(0.8*len(df))
 train = df.iloc[0:train_size,:]
@@ -152,55 +155,165 @@ test = df.iloc[train_size:, :]
 options = sider.multiselect(
     'Models',options=["ARIMA", "SARIMA", "XGBoost"])
 models = {}
+model_inv = {}
 for option in options:
     if option == "ARIMA":
-        st.divider()
         sider.divider()
         sider.subheader("ARIMA")
-        st.subheader("ARIMA")
+        func = sider.selectbox("Apply ", options=["None", "log", "sqrt"], index= 0, key="arima")
+        if func == "None":
+            df["X_ARIMA"]= df["data"]
+            model_inv["ARIMA"] = lambda x :x
+        elif func == "log":
+            try:
+                df["X_ARIMA"] = np.log(df["data"])
+                model_inv["ARIMA"] = np.exp
+            except:
+                st.error("Unable to apply log, use the default data")
+                df["X_ARIMA"]= df["data"]
+                model_inv["ARIMA"] = lambda x :x
+        elif func == "sqrt":
+            try:
+                df["X_ARIMA"] = np.sqrt(df["data"])
+                model_inv["ARIMA"] = np.square
+            except:
+                st.error("Unable to apply sqrt, use the default data")
+                df["X_ARIMA"]= df["data"]
+                model_inv["ARIMA"] = lambda x :x
+        train_size = int(0.8*len(df))
+        train = df.iloc[0:train_size,:]
+        test = df.iloc[train_size:, :]
         result = None
-        p_range = sider.slider("P Range",min_value=0, max_value=30, value=[0, 0])
-        q_range = sider.slider("Q Range",min_value=0, max_value=30, value=[0, 0])
-        ps = range(p_range[0], p_range[1]+1)
-        ds = range(d, d+1)
-        qs = range(q_range[0], q_range[1]+1)
-        if sider.toggle("Train ARIMA"):    
-            result, best_score, best_param = SARIMAXGridSearch.search(train, ps, ds, qs)
-            st.write(result, best_score, best_param)
-            sarimax_pred, conf_int = sarimax_forecast(result, steps=len(test))
-            models[option] = (result, conf_int)
-            test[option] = sarimax_pred
-        if result:
-            with st.expander("Model Diagnostics"):
-                st.write(result.plot_diagnostics())
-            with st.expander("Model Validation"):
-                lb = float(result.summary().tables[2].data[1][1])
-                jb = float(result.summary().tables[2].data[1][3])
-                st.markdown(f'<h3>Ljung-Box Test : {lb}</h3>', unsafe_allow_html=True)
-                st.markdown(f'<h3>Jarque-Box Test : {jb}</h3>', unsafe_allow_html=True)
-                color, label = ("green", "Validate") if valid_model(lb, jb) else ("red", "Reject")
-                st.markdown(f'<h3 style="color:{color};">Decision : {label}</h3>', unsafe_allow_html=True)
-                
-    if option == "SARIMA":
-        continue
-    if option == "XGBoost":
-        st.divider()
+        p_range_arima = sider.slider("P Range",min_value=0, max_value=30, value=[0, 0], key="arima_p_range")
+        q_range_arima = sider.slider("Q Range",min_value=0, max_value=30, value=[0, 0], key="arima_q_range")
+        arima_params_dict ={
+            "start_p" : p_range_arima[0],
+            "start_q" : q_range_arima[0],
+            "max_p" : p_range_arima[1],
+            "max_q" : q_range_arima[1]
+        }
+    elif option == "SARIMA" : 
+        sider.subheader("SARIMA")
+        func = sider.selectbox("Apply ", options=["None", "log", "sqrt"], index= 0, key="sarima")
+        if func == "None":
+            df["X_SARIMA"]= df["data"]
+            model_inv["SARIMA"] = lambda x :x
+        elif func == "log":
+            try:
+                df["X_SARIMA"] = np.log(df["data"])
+                model_inv["SARIMA"] = np.exp
+            except:
+                st.error("Unable to apply log, use the default data")
+                df["X_SARIMA"]= df["data"]
+                model_inv["SARIMA"] = lambda x :x
+        elif func == "sqrt":
+            try:
+                df["X_SARIMA"] = np.sqrt(df["data"])
+                model_inv["SARIMA"] = np.square
+            except:
+                st.error("Unable to apply sqrt, use the default data")
+                df["X_SARIMA"]= df["data"]
+                model_inv["SARIMA"] = lambda x :x
+        train_size = int(0.8*len(df))
+        train = df.iloc[0:train_size,:]
+        test = df.iloc[train_size:, :]
+        result = None
+        m = sider.slider("seasonal period m ",min_value=0, max_value=30, value=12)
+        p_range = sider.slider("p Range",min_value=0, max_value=30, value=[0, 0], key="sarima_p_range")
+        q_range = sider.slider("q Range",min_value=0, max_value=30, value=[0, 0], key="sarima_q_range")
+        P_range = sider.slider("P Range",min_value=0, max_value=30, value=[0, 0])
+        Q_range = sider.slider("Q Range",min_value=0, max_value=30, value=[0, 0])
+        sarima_params_dict ={
+            "seasonal":True,
+            "m":m,
+            "start_p" : p_range[0],
+            "start_q" : q_range[0],
+            "max_p" : p_range[1],
+            "max_q" : q_range[1],
+            "start_P" : P_range[0],
+            "start_Q" : Q_range[0],
+            "max_P" : P_range[1],
+            "max_Q" : q_range[1],
+        }
+    elif option == "XGBoost":
+
         sider.subheader(option)
-        st.subheader(option)
+        f1 = sider.selectbox("Apply First Transformation", options=["None", "log", "sqrt"], index= 0, key="xgboost_f1")
+        if f1 == "None":
+            df["X_XGBoost_1"]= df["data"]
+            model_inv["XGBoost"] = lambda x :x
+        elif f1 == "log":
+            try:
+                df["X_XGBoost_1"] = np.log(df["data"])
+                model_inv["XGBoost"] = np.exp
+            except:
+                st.error("Unable to apply log, use the default data")
+                df["X_XGBoost_1"]= df["data"]
+                model_inv["XGBoost"] = lambda x :x
+        elif f1 == "sqrt":
+            try:
+                df["X_XGBoost_1"] = np.sqrt(df["data"])
+                model_inv["XGBoost"] = np.square
+            except:
+                st.error("Unable to apply sqrt, use the default data")
+                df["X_XGBoost_1"]= df["data"]
+                model_inv["XGBoost"] = lambda x :x
+        f2 = sider.selectbox("Apply Second Tronsformation", options=["None","first_order_diff"], index= 0, key="xgboost_2")
+        if f2 == "None":
+            df["X_XGBoost"]= df["X_XGBoost_1"]
+            model_inv["XGBoost"] = lambda x :x
+        elif f2 == "first_order_diff":
+
+            try:
+
+                df["X_XGBoost"],xg_lags= first_order_diff(df["X_XGBoost_1"].bfill())
+                
+            except:
+                st.error("Unable to apply first_order diff, use the default data")
+        train_size = int(0.8*len(df))
+        train = df.iloc[0:train_size,:]
+        test = df.iloc[train_size:, :]
         max_depth= sider.slider("Max Depth",min_value=1, max_value=30, value=5)
         lags= sider.slider("Lags features",min_value=1, max_value=30, value=5)
         learning_rate= sider.number_input(label="Learning Rate ", min_value=0.0001, max_value=0.75, step=0.01, value=0.01)
         n_estimators= sider.number_input(label="n_estimator ", min_value=100, max_value=5000, step=100, value=1000)
-        X_train, y_train = create_features(train,lags=lags)
-        X_test, y_test = create_features(pd.concat([train.iloc[-lags:,:], test["data"]]), lags=lags)
-        if sider.toggle("Train XGBoost"):
+        
+        X_train, y_train = create_features(train,lags=lags, feature_col="X_XGBoost")
+    
+        X_test, y_test = create_features(pd.concat([train.iloc[-lags:,:], test["X_XGBoost"]]), lags=lags, feature_col="X_XGBoost")
+        
+fit = sider.button("Train Models")
+if fit:
+    for option in options:
+        if option == "ARIMA":
+            st.subheader("ARIMA")
+            result_arima  = auto_arima(train["X_ARIMA"], 
+                                 start_p=arima_params_dict["start_p"],
+                                 start_q= arima_params_dict["start_q"],
+                                 max_q = arima_params_dict["max_q"],
+                                 max_p = arima_params_dict["max_p"],
+                                 )
+            arimax_pred, conf = sarimax_forecast(result_arima, steps=len(test))
+            conf_int_arima = pd.DataFrame(conf, index = test.index, columns=['lower data', "upper data"])
+            models[option] = (result, conf_int_arima)
+            test[option] = arimax_pred
+        elif option == "SARIMA":
+            st.subheader("SARIMA")
+            result_sarima = auto_arima(train["X_SARIMA"], **sarima_params_dict)
+            sarimax_pred, conf = sarimax_forecast(result_sarima, steps=len(test))
+            conf_int_sarima = pd.DataFrame(conf, index = test.index, columns=['lower data', "upper data"])
+            models[option] = (result, conf_int_sarima)
+            test[option] = sarimax_pred
+        elif option =="XGBoost":
+            st.subheader("SARIMA")
             model_xgb = xgboost(X_train, y_train, max_depth=max_depth, learning_rate=learning_rate, n_estimators=n_estimators)
             test[option] = model_xgb.predict(X_test)
             xgb_fig, xgb_ax = plt.subplots()
-            xgb.plot_importance(model_xgb, ax = xgb_ax)
+            xgb.plot_importance(model_xgb, ax = xgb_ax, max_num_features=5)
             models[option] = (model_xgb, None)
             st.pyplot(xgb_fig)
         
+        
 st.divider()
 st.subheader("Prediction")
 
@@ -211,33 +324,43 @@ for idx, option in enumerate(options):
     if option in test.columns:
         c = None
         if option=="ARIMA":
-            c = conf_int
+            c = conf_int_arima
+        if option=="SARIMA":
+            c = conf_int_sarima
         with pred_tabs[idx]:
-            fig  = plotForcast(df, test[option], confint=c)
+            fig  = plotForcast(df[f"X_{option}"], test[option], confint=c)
             st.plotly_chart(fig)
 
 
 # Model error
 errors = {}
-metric_labels = ["MAE", "MAPE", "RMSE"]
+metric_labels = ["MAE","MAPE", "RMSE"]
 errors["Model"] = []
 errors["Type"] = []
 errors["error"] = []
 
 for option in options:
     if option in test.columns:
-        mae = mean_absolute_error(test["data"], test[option])
-        mape = mean_absolute_percentage_error(test["data"], test[option])
-        rmse = root_mean_squared_error(test["data"], test[option])
+        if option == "XGBoost" and f2 == "first_order_diff":
+            first = test["X_XGBoost_1"].iloc[:xg_lags].values
+            inv  = model_inv["XGBoost"](
+                    inverse_first_order_diff(test[option], xg_lags,first))
+        else:
+            inv = model_inv[option](test[option])
+        mae = mean_absolute_error(test["data"],inv )
+        mape = mean_absolute_percentage_error(test["data"], inv)
+        rmse = root_mean_squared_error(test["data"], inv)
         errors["Model"].extend([option]*len(metric_labels))
         errors["Type"].extend(metric_labels)
-        errors["error"].extend([mae, mape, rmse])
-sider.divider()
+        errors["error"].extend([mae,mape, rmse])
 
-if sider.toggle("Compare Models"):
+if fit:
     st.divider()
     st.subheader("Compare Models Errors")
     errors_df = pd.DataFrame(errors)
-    erro_fig, ax = plt.subplots()
-    sns.barplot(data = errors_df, y="error", x = "Type",hue="Model")
+    erro_fig, ax= plt.subplots(nrows=2)
+    sns.barplot(data = errors_df[errors_df["Type"].isin(["MAE", "RMSE"])],
+                y="error", x = "Type",hue="Model", ax=ax[0], width=0.4)
+    sns.barplot(data = errors_df[errors_df["Type"].isin(["MAPE"])],
+                y="error", x = "Type",hue="Model", width=0.2,ax=ax[1])
     st.pyplot(erro_fig)

models/arima.py

@@ -4,6 +4,7 @@ import statsmodels.api as sm
 from itertools import product
 from scipy import stats
 from stqdm import stqdm
+import pmdarima as pm
 """
 @st.cache_data
 def gridSearch(endog, order_ls:list, d= 0):
@@ -60,22 +61,26 @@ def valid_model(lb, jb):
 
 #@st.cache_data
 def sarimax_forecast(model, steps):
-    forecat = model.get_forecast(steps=steps)
-    predicted = forecat.predicted_mean
-    confint = forecat.conf_int()
-    return predicted, confint
+    forecat, confint = model.predict(n_periods=steps, return_conf_int=True)
+    
+    return forecat, confint
 
 @st.cache_data
-def auto_arima():
-    model = pm.auto_arima(data_actual, 
-                      m=12,               # frequency of series                      
+def auto_arima(endog, m=0, seasonal=False, d=None,D=None, start_p=0, start_q=0, start_P=0, start_Q=0, max_p=12, max_q=12, max_P=0, max_Q=0):
+    print(m, seasonal, d, D, start_p,start_q, max_p, max_q)
+    return pm.auto_arima(endog, 
+                      m=m,               # frequency of series                      
                       seasonal=seasonal,  # TRUE if seasonal series
-                      d=None,             # let model determine 'd'
+                      d=d,             # let model determine 'd'
                       test='adf',         # use adftest to find optimal 'd'
-                      start_p=0, start_q=0, # minimum p and q
-                      max_p=12, max_q=12, # maximum p and q
-                      D=None,             # let model determine 'D'
+                      start_p=start_p, start_q=start_q, # minimum p and q
+                      max_p=max_p, max_q=max_q, # maximum p and q
+                      start_P=start_P,
+                      start_Q= start_Q,
+                      max_P=max_P,
+                      max_Q = max_Q,
+                      D=D,             # let model determine 'D'
                       trace=True,
                       error_action='ignore',  
-                      suppress_warnings=True, 
+                      suppress_warnings=True,
                       stepwise=True)

utils/dataprocess.py

@@ -32,11 +32,12 @@ def numberOfDiff(y):
         d = d+1
     return d, diff
 
-@st.cache_data
-def create_features(df, lags = 1):
+
+def create_features(df, lags = 1, feature_col="data"):
     """
     Creates time series features from datetime index
     """
+    df =df[feature_col].to_frame()
     df['date'] = df.index
     df['hour'] = df['date'].dt.hour
     df['dayofweek'] = df['date'].dt.dayofweek
@@ -46,8 +47,8 @@ def create_features(df, lags = 1):
     df['dayofyear'] = df['date'].dt.dayofyear
     df['dayofmonth'] = df['date'].dt.day
     for i in range(1, lags):
-        df[f'lag_{i}'] = df["data"].shift(i)
-    df["target"] = df["data"].shift(lags)
+        df[f'lag_{i}'] = df[feature_col].shift(i)
+    df["target"] = df[feature_col].shift(lags)
     df.dropna(inplace=True)
     y = df[["target"]]
     X = df.drop(columns=["target", "date"])
@@ -72,6 +73,19 @@ def apply_exponential_average(data):
 
 def first_order_diff(df):
     lags = st.sidebar.slider("lags", min_value=1, max_value=30, value=1)
-    return df.diff(lags)
+    return df.diff(lags), lags
+def inverse_first_order_diff(df, lags, first):
+    df =df
+    l = []
+    l.extend(first)
+    ls = df.values
+
+    for i in range(lags, len(ls)):
+        l.append(ls[i]+l[i-lags])
+    df = l
+    return df
+
+
+
 def second_order_diff(df):
     return df.diff().diff()

utils/graphics.py

@@ -57,7 +57,7 @@ def plotForcast(df, pred, confint):
     fig.add_trace(
         go.Line(
             x = df.index,
-            y = df.data,
+            y = df,
             name= "Observed"
         )
     )