import pandas as pd
import tsfresh
import pprint
import numpy as np
import warnings
from timeit import default_timer as timer

from sklearn.ensemble import VotingClassifier


warnings.filterwarnings("ignore") #ignore warnings caused by


#################################################################
#                                                               #
#               malicious csv files import                      #
#                                                               #
#################################################################


df1 = pd.read_csv('malicious/WebOS_binary.csv') #
df2 = pd.read_csv('malicious/Server_Binary.csv') #
df3 = pd.read_csv('malicious/Raspberry_Webmine_Robust.csv')
df4 = pd.read_csv('malicious/Raspberry_Binary.csv') #
df5 = pd.read_csv('malicious/Raspberry_Webmine_Aggressive.csv')
df6 = pd.read_csv('malicious/Raspberry_WebminePool_Aggressive.csv')
df7 = pd.read_csv('malicious/Server_WebminePool_Aggressive.csv') #
df32 = pd.read_csv('malicious/Server_WebminePool_Robust.csv') #
df33 = pd.read_csv('malicious/Raspberry_WebminePool_Stealthy.csv') #
df34 = pd.read_csv('malicious/Raspberry_WebminePool_Robust.csv') #
df35 = pd.read_csv('malicious/Desktop_WebminePool_Aggressive.csv') #


#################################################################
#                                                               #
#               benign-1 csv files import                         #
#                                                               #
#################################################################

df8 = pd.read_csv('benign-1/interactive_01.csv') #
df9 = pd.read_csv('benign-1/interactive_02.csv') #
df10 = pd.read_csv('benign-1/interactive_03.csv') #
df11 = pd.read_csv('benign-1/interactive_04.csv') #
df12 = pd.read_csv('benign-1/interactive_05.csv') #
df13 = pd.read_csv('benign-1/interactive_06.csv') #
df14 = pd.read_csv('benign-1/web_1page_01.csv') #
df15 = pd.read_csv('benign-1/web_1page_02.csv') #
df16 = pd.read_csv('benign-1/web_1page_03.csv') #
df17 = pd.read_csv('benign-1/web_1page_04.csv') #
df18 = pd.read_csv('benign-1/web_1page_05.csv') #
df19 = pd.read_csv('benign-1/bulk_xs_04.csv') #
df20 = pd.read_csv('benign-1/bulk_xs_05.csv') #
df21 = pd.read_csv('benign-1/video_180s480p_01.csv') #
df22 = pd.read_csv('benign-1/video_180s480p_02.csv') #
df23 = pd.read_csv('benign-1/video_x1_04.csv') #
df24 = pd.read_csv('benign-1/web_multiple_04.csv') #
df25 = pd.read_csv('benign-1/bulk_xs_01.csv') #
df26 = pd.read_csv('benign-1/bulk_xs_09.csv') #
df27 = pd.read_csv('benign-1/bulk_xs_06.csv') #
df28 = pd.read_csv('benign-1/bulk_xs_03.csv') #
df29 = pd.read_csv('benign-1/web_multiple_03.csv') #
df30 = pd.read_csv('benign-1/web_multiple_05.csv') #
df31 = pd.read_csv('benign-1/web_multiple_06.csv') #

df_results = pd.DataFrame()

# For WebOS = 18:56:80:17:d0:ef
index_names = df1[((df1['HW_dst'] != '18:56:80:17:d0:ef') & (df1['Hw_src'] != '18:56:80:17:d0:ef'))].index
df1.drop(index_names, inplace = True)

# Big_Server_Monero_mining_data = a4:bb:6d:ac:e1:fd

index_names = df2[((df2['HW_dst'] != 'a4:bb:6d:ac:e1:fd') & (df2['Hw_src'] != 'a4:bb:6d:ac:e1:fd'))].index
df2.drop(index_names, inplace = True)

# ege_data_rasberry = dc:a6:32:67:66:4b

index_names = df3[((df3['HW_dst'] != 'dc:a6:32:67:66:4b') & (df3['Hw_src'] != 'dc:a6:32:67:66:4b'))].index
df3.drop(index_names, inplace = True)

# Rasberry_binary_monero_mining = dc:a6:32:68:35:8a

index_names = df4[((df4['HW_dst'] != 'dc:a6:32:68:35:8a') & (df4['Hw_src'] != 'dc:a6:32:68:35:8a'))].index
df4.drop(index_names, inplace = True)

# Rasberry_network_data_2 = dc:a6:32:67:66:4b

index_names = df5[((df5['HW_dst'] != 'dc:a6:32:67:66:4b') & (df5['Hw_src'] != 'dc:a6:32:67:66:4b'))].index
df5.drop(index_names, inplace = True)

# Rasberry-Webmine = dc:a6:32:67:66:4b
index_names = df6[((df6['HW_dst'] != 'dc:a6:32:67:66:4b') & (df6['Hw_src'] != 'dc:a6:32:67:66:4b'))].index
df6.drop(index_names, inplace = True)

# Server_Webmine_Network_data = a4:bb:6d:ac:e1:fd

index_names = df7[((df7['HW_dst'] != 'a4:bb:6d:ac:e1:fd') & (df7['Hw_src'] != 'a4:bb:6d:ac:e1:fd'))].index
df7.drop(index_names, inplace = True)

# Server_%50_Mining = a4:bb:6d:ac:e1:fd

index_names = df32[((df32['HW_dst'] != 'a4:bb:6d:ac:e1:fd') & (df32['Hw_src'] != 'a4:bb:6d:ac:e1:fd'))].index
df32.drop(index_names, inplace = True)

# Rasberry_webmine_%10 = dc:a6:32:67:66:4b

index_names = df33[((df33['HW_dst'] != 'dc:a6:32:67:66:4b') & (df33['Hw_src'] != 'dc:a6:32:67:66:4b'))].index
df33.drop(index_names, inplace = True)

# Rasberry_webmine_%50 = dc:a6:32:68:35:8a

index_names = df34[((df34['HW_dst'] != 'dc:a6:32:68:35:8a') & (df34['Hw_src'] != 'dc:a6:32:68:35:8a'))].index
df34.drop(index_names, inplace = True)

# Desktop_Webmine_%100 = dc:a6:32:68:35:8a

index_names = df35[((df35['HW_dst'] != 'd8:3b:bf:8f:ba:ba') & (df35['Hw_src'] != 'd8:3b:bf:8f:ba:ba'))].index
df35.drop(index_names, inplace = True)

df1.insert(7, "Is_malicious", 1)
df2.insert(7, "Is_malicious", 1)
df3.insert(7, "Is_malicious", 1)
df4.insert(7, "Is_malicious", 1)
df5.insert(7, "Is_malicious", 1)
df6.insert(7, "Is_malicious", 1)
df7.insert(7, "Is_malicious", 1)

# ========================================================

df8.insert(7, "Is_malicious", 0)
df9.insert(7, "Is_malicious", 0)
df10.insert(7, "Is_malicious", 0)
df11.insert(7, "Is_malicious", 0)
df12.insert(7, "Is_malicious", 0)
df13.insert(7, "Is_malicious", 0)
df14.insert(7, "Is_malicious", 0)
df15.insert(7, "Is_malicious", 0)
df16.insert(7, "Is_malicious", 0)
df17.insert(7, "Is_malicious", 0)
df18.insert(7, "Is_malicious", 0)
df19.insert(7, "Is_malicious", 0)
df20.insert(7, "Is_malicious", 0)
df21.insert(7, "Is_malicious", 0)
df22.insert(7, "Is_malicious", 0)
df23.insert(7, "Is_malicious", 0)
df24.insert(7, "Is_malicious", 0)


# ========================================================

df25.insert(7, "Is_malicious", 0)
df26.insert(7, "Is_malicious", 0)
df27.insert(7, "Is_malicious", 0)
df28.insert(7, "Is_malicious", 0)
df29.insert(7, "Is_malicious", 0)
df30.insert(7, "Is_malicious", 0)
df31.insert(7, "Is_malicious", 0)

df32.insert(7, "Is_malicious", 1)
df33.insert(7, "Is_malicious", 1)
df34.insert(7, "Is_malicious", 1)
df35.insert(7, "Is_malicious", 1)


def run_process(a, b, x):
    df_malicious = a.copy()
    df_benign = b.copy()

    from tsfresh.utilities.dataframe_functions import impute
    df_malicious.reset_index(drop=True, inplace=True)  # reset index
    df_malicious['id'] = np.floor(df_malicious.index.array / 10)
    df_benign.reset_index(drop=True, inplace=True)  # reset index
    df_benign['id'] = np.floor(df_benign.index.array / 10)

    tf1 = tsfresh.extract_features(df_malicious, impute_function=impute, column_kind='Is_malicious',
                                   column_id='id', column_sort="Time", column_value="Length")
    tf1['class'] = 1

    tf2 = tsfresh.extract_features(df_benign, impute_function=impute, column_kind='Is_malicious',
                                   column_id='id', column_sort="Time", column_value="Length")
    tf2['class'] = 0

    tf2.columns = tf1.columns

    features = pd.concat([tf1, tf2])

    features2 = features.copy()
    features2.reset_index(drop=True, inplace=True)

    y = pd.Series(data=features2['class'], index=features2.index)


    from tsfresh.feature_selection.relevance import calculate_relevance_table

    relevance_table = calculate_relevance_table(features2, y)
    relevance_table = relevance_table[relevance_table.relevant]
    relevance_table.sort_values("p_value", inplace=True)

    relevance_table

    best_features = relevance_table[relevance_table['p_value'] <= 0.05]

    df_ML = pd.DataFrame()
    for pkt in best_features:
        df_ML[best_features.feature] = features[best_features.feature]

    df_ML = pd.DataFrame()


    df_ML[best_features.feature] = features[best_features.feature]

    final = ML_Process(df_ML, x)

    return final


def ML_Process(df_ML, x):
    df_results = x.copy()
    print('let the ml starts')

    from sklearn.preprocessing import LabelEncoder

    # X = df_finalized[['Time', 'Length','Protocol']].values
    X = df_ML.drop('class', axis=1).to_numpy()
    # y = df_finalized[['Is_malicious']]
    y = df_ML['class'].to_numpy()

    # print(X,y)

    from sklearn.model_selection import train_test_split
    Le = LabelEncoder()
    for i in range(len(X[0])):
        X[:, i] = Le.fit_transform(X[:, i])

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=8675309)

    from sklearn.linear_model import LogisticRegression
    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.svm import SVC

    from sklearn.naive_bayes import GaussianNB

    from sklearn import model_selection

    from sklearn.metrics import classification_report

    import pandas as pd
    y_train = y_train.ravel()
    dfs = []
    models1 = [
        ('LogReg', LogisticRegression()),
        ('KNN', KNeighborsClassifier(weights='distance')),  # 启用距离加权，但这不是概率预测
        ('SVM', SVC(probability=True)),  # 启用概率预测
        ('GNB', GaussianNB()),
    ]
    models = [
        ('LogReg', LogisticRegression()),
        ('KNN', KNeighborsClassifier()),
        ('SVM', SVC()),
        ('GNB', GaussianNB()),
        ('voting_clf',VotingClassifier(
        estimators=models1,
        voting='soft'  # 'soft' 使用概率的平均，'hard' 使用多数投票
    ))
    ]

    target_names = ['malignant', 'benign']

    results = []
    names = []
    scoring = ['accuracy', 'precision_weighted', 'recall_weighted', 'f1_weighted', 'roc_auc']

    for name, model in models:
        kfold = model_selection.KFold(n_splits=5, shuffle=True, random_state=90210)
        cv_results = model_selection.cross_validate(model, X_train, y_train, cv=kfold,
                                                    scoring=scoring)

        clf = model.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        print(name)
        print(classification_report(y_test, y_pred, target_names=target_names))
        pprint.pprint(cv_results)
    #     results.append(cv_results)
    #     names.append(name)
    #     this_df = pd.DataFrame(cv_results)
    #     this_df['model'] = name
    #     dfs.append(this_df)
    #     df_resulta = df_results.append(dfs)
    #     final = pd.concat(dfs, ignore_index=True)
    #     pprint.pprint(final)
    # final.to_csv('In-Browser.csv', index=False)
    #return (final)
if __name__ == '__main__':
    df_malicious = pd.concat([df1, df2, df4])
    df_benign = pd.concat([df12, df13, df15, df17, df22, df23, df27, df28, df29])

    print("malicious: {}".format(len(df_malicious)))
    print("benign: {}".format(len(df_benign)))

    print("{} NAN in malicious!".format(len(df_malicious[df_malicious.isna().any(axis=1)])))
    print("{} NAN in benign!".format(len(df_benign[df_benign.isna().any(axis=1)])))

    df_malicious = df_malicious.dropna()
    df_benign = df_benign.dropna()

    print("After droppping NAN rows: ")
    print("malicious: {}".format(len(df_malicious)))
    print("benign: {}".format(len(df_benign)))

    start = timer()

    results_Host_s3 = run_process(df_malicious, df_benign, df_results)

    end = timer()
    print(end - start)      数据处理是怎么样的，使用了哪些归一化吗