Kaggle competition: Microsoft Malware Prediction
<SOS> metric: AUC Kaggle data source: microsoft-malware-prediction<train_model>
LABELS = ["HasDetections"]
train_processed = data[:num_train]
test_processed = data[num_train:]
print("Train :",train.shape)
 print("Test:",test.shape )<feature_engineering>
predict_list = []
predict_list.append(pd.read_csv(".. /input/lightgbm/submission_lgbm.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/malware-predictions-3500-trees/submission0.72968.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/detecting-malwares-with-ftrl-proximal/submission.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/outputs-for-microsoft/submission_ashish_kfold.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_5.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_6.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_7.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_8.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv")[LABELS].values )
train["revenue"] = [np.log(num)for num in train["revenue"]]
len(train["revenue"] )<feature_engineering>
print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('super_blend.csv', index=False )
train_processed["revenue"] = train["revenue"].values
train = train_processed
test = test_processed<feature_engineering>
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from math import sqrt
from sklearn import metrics
def prepare_data_frame(dataframe, target):
df = dataframe.copy()
df['Open Date Year'] = df['Open Date'].dt.year
df['Open Date Month'] = df['Open Date'].dt.month
df['Open Date Day'] = df['Open Date'].dt.day
all_diff = []
for date in df["Open Date"]:
diff = dt.now() - date
all_diff.append(int(diff.days/1000))
df['Days_from_open'] = pd.Series(all_diff)
 df = df.drop(['Open Date'], axis=1)
 if target in df.columns:
tar = df[target]
df = df.drop([target], axis=1)
 else:
tar = None
num_vars = df.select_dtypes([np.number] ).columns.tolist()
cat_vars = df.select_dtypes(include='object' ).columns.tolist()
df[cat_vars] = df[cat_vars].apply(preprocessing.LabelEncoder().fit_transform)
 return(df,tar )<prepare_x_and_y>
df_base0 = pd.read_csv(".. /input/lightgbm/submission_lgbm.csv",names=["MachineIdentifier","HasDetections0"], skiprows=[0],header=None)
 df_base1 = pd.read_csv(".. /input/malware-predictions-3500-trees/submission0.72968.csv",names=["MachineIdentifier","HasDetections1"], skiprows=[0],header=None)
 df_base2 = pd.read_csv(".. /input/detecting-malwares-with-ftrl-proximal/submission.csv",names=["MachineIdentifier","HasDetections2"], skiprows=[0],header=None)
 df_base3 = pd.read_csv(".. /input/outputs-for-microsoft/submission_ashish_kfold.csv",names=["MachineIdentifier","HasDetections3"], skiprows=[0],header=None)
 df_base4 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_5.csv",names=["MachineIdentifier","HasDetections4"], skiprows=[0],header=None)
 df_base5 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_6.csv",names=["MachineIdentifier","HasDetections5"], skiprows=[0],header=None)
 df_base6 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_7.csv",names=["MachineIdentifier","HasDetections6"], skiprows=[0],header=None)
 df_base7 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_8.csv",names=["MachineIdentifier","HasDetections7"], skiprows=[0],header=None)
 df_base8 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv",names=["MachineIdentifier","HasDetections8"], skiprows=[0],header=None )
X_train,y_train = prepare_data_frame(train,'revenue')
 X_test,y_test = prepare_data_frame(test,'revenue' )<set_options>
df_base = pd.merge(df_base0,df_base1,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base2,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base3,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base4,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base5,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base6,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base7,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base8,how='inner',on='MachineIdentifier' )
warnings.filterwarnings('ignore' )<train_on_grid>
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))
def get_reg_mse(model,in_parameters):
my_model_1 = model
clf = GridSearchCV(my_model_1, in_parameters, cv=2, scoring='neg_mean_squared_error')
 clf.fit(X_train, y_train)
 print('best_params_',clf.best_params_)
 mae_1 = clf.best_score_ * -1
return mae_1<choose_model_class>
df_base['HasDetections'] = df_base.iloc[:,1:].mean(axis=1)
 df_base[['MachineIdentifier','HasDetections']].to_csv("Bestoutput.csv",index=False )
def baseline_model() :
dim = len(X_train.columns.tolist())
 if not isinstance(dim, int):
return 0
model = tf.keras.Sequential([
layers.Dense(dim, input_dim=dim, kernel_initializer='normal', activation='relu'),
layers.Dense(int(round(dim/2)) , kernel_initializer='normal', activation='relu'),
layers.Dense(int(round(dim/4)) , kernel_initializer='normal', activation='relu'),
layers.Dense(int(round(dim/8)) , kernel_initializer='normal', activation='relu'),
layers.Dense(1, kernel_initializer='normal')
 ])
 model.compile(optimizer='adam',
loss='mse',
metrics=['mse'])
 return model<train_on_grid>
df_base0 = pd.read_csv(".. /input/lightgbm/submission_lgbm.csv",names=["MachineIdentifier","HasDetections0"], skiprows=[0],header=None)
 df_base1 = pd.read_csv(".. /input/malware-predictions-3500-trees/submission0.72968.csv",names=["MachineIdentifier","HasDetections1"], skiprows=[0],header=None)
 df_base2 = pd.read_csv(".. /input/detecting-malwares-with-ftrl-proximal/submission.csv",names=["MachineIdentifier","HasDetections2"], skiprows=[0],header=None)
 df_base3 = pd.read_csv(".. /input/outputs-for-microsoft/submission_ashish_kfold.csv",names=["MachineIdentifier","HasDetections3"], skiprows=[0],header=None)
 df_base8 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv",names=["MachineIdentifier","HasDetections8"], skiprows=[0],header=None )
def get_reg_keras_mse(in_parameters):
my_model_1 = tf.keras.wrappers.scikit_learn.KerasRegressor(build_fn=baseline_model, verbose=1)
 reg = GridSearchCV(my_model_1, in_parameters, cv=2, scoring='neg_mean_squared_error')
 X_ktrain = StandardScaler().fit_transform(X_train.values)
 arr = sav_train['rev_save'].values
y_ktrain = StandardScaler().fit_transform(arr[:, np.newaxis])
 reg.fit(X_ktrain, y_train.values)
 print('best_params_',reg.best_params_)
 mse_1 = reg.best_score_ * -1
return mse_1
def df_to_dataset(dataframe, shuffle=True, batch_size=32, target_name='target'):
dataframe = dataframe.copy()
labels = dataframe.pop(target_name)
 ds = tf.data.Dataset.from_tensor_slices(( dict(dataframe), labels))
if shuffle:
ds = ds.shuffle(buffer_size=len(dataframe))
ds = ds.batch(batch_size)
 return ds
def input_fn(features, labels, training=True, batch_size=32):

 dataset = tf.data.Dataset.from_tensor_slices(( dict(features), labels))
if training:
dataset = dataset.shuffle(buffer_size=len(features))
return dataset.batch(batch_size )<load_from_csv>
df_base = pd.merge(df_base0,df_base1,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base2,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base3,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base8,how='inner',on='MachineIdentifier' )
samp = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/sampleSubmission.csv')
 samp.head()<normalization>
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))
<data_type_conversions>
df_base['HasDetections'] = df_base.iloc[:,1:].mean(axis=1)
 df_base[['MachineIdentifier','HasDetections']].to_csv("Bestoutput1.csv",index=False )
<save_to_csv>
double_blend = df_base['HasDetections'] * 0.5 + submission['HasDetections'] * 0.5
sample = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 sample.head()
<load_from_csv><EOS> 
sample['HasDetections'] = double_blend
sample.to_csv('Bouble_blend.csv', index=False )