import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Convolution1D, Flatten
from keras.wrappers.scikit_learn import KerasClassifier
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.models import load_model
from sklearn.model_selection import StratifiedKFold, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.tree import DecisionTreeClassifier, export_graphviz
import pydot
from subprocess import call

# Load the data
data = pd.read_csv("normalfee_reduce.csv")
data.fillna(data.mean(),inplace=True)


# Data preprocessing
X = data.drop(["Diff_F"], axis=1)
y = data["Diff_F"]
scaler = StandardScaler()
X = scaler.fit_transform(X)

# Define a function for building the DNN model
def build_dnn_model():
    model = Sequential()
    model.add(Dense(units=16, activation="relu", input_shape=(X.shape[1],)))
    model.add(Dense(units=1, activation="sigmoid"))
    model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
    return model

# Define a function for building the CNN model
def build_cnn_model():
    model = Sequential()
    model.add(Convolution1D(filters=16, kernel_size=3, activation="relu", input_shape=(X.shape[1],1)))
    model.add(Flatten())
    model.add(Dense(units=16, activation="relu"))
    model.add(Dense(units=1, activation="sigmoid"))
    model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
    return model

# Define a function for building the CHAID-DNN model
def build_chaid_dnn_model():
    model = Sequential()
    model.add(Dense(units=16, activation="relu", input_shape=(X_chaid.shape[1],)))
    model.add(Dense(units=1, activation="sigmoid"))
    model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
    return model

# Define a function for building the CHAID-CNN model
def build_chaid_cnn_model():
    model = Sequential()
    model.add(Convolution1D(filters=16, kernel_size=3, activation="relu", input_shape=(X_chaid.shape[1],1)))
    model.add(Flatten())
    model.add(Dense(units=16, activation="relu"))
    model.add(Dense(units=1, activation="sigmoid"))
    model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
    return model

# Define a function for using CHAID to select important variables
def chaid_select(data):
    model = DecisionTreeClassifier(criterion="entropy")
    model.fit(data.drop(["Diff_F"], axis=1), data["Diff_F"])
    export_graphviz(model, out_file="tree.dot", feature_names=data.drop(["Diff_F"], axis=1).columns.values)
    call(["dot", "-Tpng", "tree.dot", "-o", "tree.png", "-Gdpi=600"])
    chaid_features = model.tree_.feature[np.where(model.tree_.children_left != -1)]
    return data.iloc[:, chaid_features]

# Use CHAID to select important variables
X_chaid = chaid_select(data)

# Train models using k-fold cross-validation with hyperparameter search
models = []
model_names = ["Diff_DNN", "Diff_CNN", "Diff_DNNCHAID", "Diff_CNNCHAID"]
for i, model_name in enumerate(model_names):
    if ("DNN" in model_name):
        clf = KerasClassifier(build_fn=build_dnn_model, epochs=50, batch_size=64, verbose=0)
    elif ("CNN" in model_name):
        X_reshape = np.reshape(X, (X.shape[0], X.shape[1], 1))
        clf = KerasClassifier(build_fn=build_cnn_model, epochs=50, batch_size=64, verbose=0)
    else:
        clf = KerasClassifier(build_fn=build_dnn_model, epochs=50, batch_size=64, verbose=0)
        X_chaid_reshape = np.reshape(X_chaid, (X_chaid.shape[0], X_chaid.shape[1], 1))

    grid_search = GridSearchCV(estimator=clf, param_grid={"epochs": [50, 100], "batch_size": [32, 64]}, cv=5, n_jobs=-1)
    early_stopping = EarlyStopping(monitor="val_loss", patience=5, mode="min", verbose=1)
    model_checkpoint = ModelCheckpoint(model_name + ".h5", monitor="val_loss", mode="min", verbose=1, save_best_only=True)
    callbacks = [early_stopping, model_checkpoint]
    if ("DNN" in model_name):
        grid_result = grid_search.fit(X, y, validation_split=0.2, callbacks=callbacks)
        best_model = load_model(model_name + ".h5")
        y_pred = best_model.predict(X)
        data[model_name] = y_pred
    else:
        grid_result = grid_search.fit(X_reshape, y, validation_split=0.2, callbacks=callbacks)
        best_model = load_model(model_name + ".h5")
        y_pred = best_model.predict(X_reshape)
        data[model_name] = y_pred

data.to_csv("output.csv", index=False)
print("Output saved as output.csv.")