Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<normalization>
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import StratifiedKFold, KFold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm
from sklearn.covariance import EmpiricalCovariance
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn import linear_model
import sympy
import numpy as np
from sklearn.covariance import OAS
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
train_columns = train_X.columns
n_fold = 5<split>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=3, threshold=0.6, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=6, init_params='kmeans',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,
means_init=ms, precisions_init=ps, random_state=666)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:3].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:3].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))
%%time
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 oof = np.zeros(len(train_X))
train_score = []
fold_idxs = []
predictions = np.zeros(len(test_X))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_X,train_y.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 fold_idxs.append(val_idx)
 X_tr, X_val = train_X[train_columns].iloc[trn_idx], train_X[train_columns].iloc[val_idx]
y_tr, y_val = train_y.iloc[trn_idx], train_y.iloc[val_idx]
model = CatBoostRegressor(n_estimators=25000, verbose=-1, objective="MAE", loss_function="MAE", boosting_type="Ordered", task_type="GPU")
 model.fit(X_tr,
y_tr,
eval_set=[(X_val, y_val)],
verbose=2500,
early_stopping_rounds=500)
 oof[val_idx] = model.predict(X_val)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += model.predict(test_X[train_columns])/ folds.n_splits
train_score.append(model.best_score_['learn']["MAE"])
 cv_score = mean_absolute_error(train_y, oof)
 print(cv_score )<save_to_csv>
oof_0 = oof
preds_0 = preds
today = str(datetime.date.today())
 submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 cat_saved=pd.DataFrame(oof,columns=['oof'])
 cat_saved.to_csv('cat_oof.csv',index=False)
 submission["time_to_failure"] = predictions
submission.to_csv(f'CatBoost_submission_{cv_score:.3f}.csv', index=False)
 submission.head()<import_modules>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=4, threshold=0.6, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=8, init_params='kmeans',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,
means_init=ms, precisions_init=ps, random_state=666)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:4].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:4].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))
import numpy as np
import pandas as pd
from keras.layers import *
from keras.models import *
from keras.optimizers import *
from keras.preprocessing.image import ImageDataGenerator
import os<load_from_csv>
oof_1 = oof
preds_1 = preds
def load_datasets(validation_split=0.2):
def load_data(filepath, test=False):
df = pd.read_csv(filepath)
 data = df.values
if not test:
x_data = data[:, 1:]
y_data = data[:, 0]
else:
x_data = data
y_data = np.empty(0)
 x_data = x_data.astype(np.float32)/ 255.0
x_data = np.reshape(x_data,(x_data.shape[0], 28, 28, 1))
return x_data, y_data
base_folder = '.. /input/'
x_train = []
y_train = []
x_test = []
x_train, y_train = load_data(base_folder + 'train.csv')
 x_test, _ = load_data(base_folder + 'test.csv', test=True)
 indexes = np.random.permutation(range(x_train.shape[0]))
x_train = x_train[indexes]
y_train = y_train[indexes]
split_index = int(x_train.shape[0] * validation_split)
 x_val = x_train[:split_index]
y_val = y_train[:split_index]
x_train = x_train[split_index:]
y_train = y_train[split_index:]
return x_train, y_train, x_val, y_val, x_test
def build_model() :
kernel_size = 3
filters = 16
image_size = 28
classes = 10
model_inputs = Input(( 28, 28,1))
processed = model_inputs
while image_size != 3:
processed = Conv2D(filters, kernel_size, padding='same' )(processed)
 processed = LeakyReLU()(processed)
 processed = Conv2D(filters, kernel_size, padding='same' )(processed)
 processed = LeakyReLU()(processed)
 processed = MaxPooling2D()(processed)
 image_size = int(image_size / 2)
 filters *= 2
processed = GlobalAveragePooling2D()(processed)
 processed = Dropout(0.5 )(processed)
 processed = Dense(10, activation='softmax' )(processed)
 model = Model(model_inputs, processed)
 model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 print(model.summary())
 return model
def save_predictions(model, x_test):
predictions = model.predict(x_test)
 predictions = np.argmax(predictions, axis=1)
 with open('output.csv', 'w')as f:
print('ImageId,Label', file=f)
 for i, prediction in enumerate(predictions):
print(str(i+1)+ ',' + str(prediction), file=f )<train_model>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
x_train, y_train = train3[train_index,:], train2.loc[train_index]['target'].values
x_train_0 = x_train[y_train==0]
x_train_1 = x_train[y_train==1]
brc = Birch(branching_factor=50, n_clusters=2, threshold=0.6, compute_labels=True)
 labels_0 = brc.fit_predict(x_train_0)
 labels_1 = brc.fit_predict(x_train_1)
 zero_mean = []
zero_cov = []
for l in np.unique(labels_0):
model = OAS()
model.fit(x_train_0[labels_0==l])
 p = model.precision_
m = model.location_
zero_mean.append(m)
 zero_cov.append(p)
 one_mean = []
one_cov = []
for l in np.unique(labels_1):
model = OAS()
model.fit(x_train_1[labels_1==l])
 p = model.precision_
m = model.location_
one_mean.append(m)
 one_cov.append(p)
 ms = np.stack(zero_mean + one_mean)
 ps = np.stack(zero_cov + one_cov)
 gm = GaussianMixture(n_components=4, init_params='kmeans',
covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,
means_init=ms, precisions_init=ps, random_state=666)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:, 0:2].mean(axis=1)
 preds[idx2] += gm.predict_proba(test3)[:, 0:2].mean(axis=1)/ skf.n_splits
print('AUC ', i, roc_auc_score(1- train2['target'], oof[idx1]))
auc = roc_auc_score(1 - train['target'],oof)
 print('QDA scores CV =',round(auc,5))
x_train, y_train, x_val, y_val, x_test = load_datasets()
print(x_train.shape, y_train.shape, x_val.shape, y_val.shape, x_test.shape)
 datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range = 0.1)
 datagen.fit(x_train )<train_model>
oof_2 = oof
preds_2 = preds
model = build_model()
batch_size = 32
model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_val, y_val),
steps_per_epoch=x_train.shape[0] / batch_size, epochs=15 )<predict_on_test>
train['y_0'] = oof_0
test['y_0'] = preds_0
train['y_1'] = oof_1
test['y_1'] = preds_1
train['y_2'] = oof_2
test['y_2'] = preds_2
save_predictions(model, x_test )<set_options>
oof_features = ['y_0', 'y_1', 'y_2']
%matplotlib inline
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore')
<feature_engineering>
for m in sorted(train['wheezy-copper-turtle-magic'].unique()):
idx_tr =(train['wheezy-copper-turtle-magic']==m)
 idx_te =(test['wheezy-copper-turtle-magic']==m)
 oofs = []
preds = []
kf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 oof_preds = np.zeros(( len(train[idx_tr][oof_features]), 1))
test_preds = np.zeros(( len(test[idx_te][oof_features]), 1))
for idx,(train_index, valid_index)in enumerate(kf.split(train[idx_tr][oof_features], train[idx_tr]['target'])) :
y_train, y_valid = train[idx_tr]['target'].iloc[train_index], train[idx_tr]['target'].iloc[valid_index]
x_train, x_valid = train[idx_tr][oof_features].iloc[train_index,:], train[idx_tr][oof_features].iloc[valid_index,:]
model = linear_model.Ridge(alpha=3)
 model.fit(x_train, y_train)
 oof_preds[valid_index, :] = model.predict(x_valid ).reshape(( -1, 1))
test_preds += model.predict(test[idx_te][oof_features] ).reshape(( -1, 1)) / 11
print('OOF AUC ', m, ' ', roc_auc_score(train[idx_tr]['target'], oof_preds))
oofs.append(( idx_tr, oof_preds))
preds.append(( idx_te, test_preds))
for ids, target in oofs:
train.loc[ids,'target_final'] = target
for ids, target in preds:
test.loc[ids,'target_final'] = target
print('OOF AUC ', roc_auc_score(train['target'], train['target_final']))
def NormalizeData(df):

 num_cols = df._get_numeric_data().columns
skew_features = df[num_cols].apply(lambda x: x.skew() ).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
df[i] = boxcox1p(df[i], boxcox_normmax(df[i] + 1))
return df<categorify>
test_preds = pd.DataFrame(np.concatenate([preds_0.reshape(-1, 1), preds_1.reshape(-1, 1), preds_2.reshape(-1, 1), 1 - test['target_final'].values.reshape(-1, 1)], axis=1))
def OrdinalsToNumerics(df):

 df['ExterQual'] = df['ExterQual'].map(lambda x: {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}.get(x, 0))
df['BsmtQual'] = df['BsmtQual'].map(lambda x: {'NA':1,'Po':2, 'Fa':3, 'TA':4, 'Gd':5, 'Ex':6}.get(x, 0))
df['KitchenQual'] = df['KitchenQual'].map(lambda x: {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}.get(x, 0))
df['GarageFinish'] = df['GarageFinish'].map(lambda x: {'NA':1, 'Unf':2, 'RFn':3, 'Fin':4}.get(x, 0))
df['FireplaceQu'] = df['FireplaceQu'].map(lambda x: {'NA':1,'Po':2, 'Fa':3, 'TA':4, 'Gd':5, 'Ex':6}.get(x, 0))
df['HeatingQC'] = df['HeatingQC'].map(lambda x: {'Po':1, 'Fa':2, 'TA':3, 'Gd':4, 'Ex':5}.get(x, 0))
df['OverallQual'] = pd.to_numeric(df['OverallQual'])
 df['ExterQual'] = pd.to_numeric(df['ExterQual'])
 df['BsmtQual'] = pd.to_numeric(df['BsmtQual'])
 df['KitchenQual'] = pd.to_numeric(df['KitchenQual'])
 df['GarageFinish'] = pd.to_numeric(df['GarageFinish'])
 df['FireplaceQu'] = pd.to_numeric(df['FireplaceQu'])
 df['HeatingQC'] = pd.to_numeric(df['HeatingQC'] )<categorify>
y_hat = test_preds.rank(axis=0, method='min' ).mul(test_preds.shape[1] * [1 / test_preds.shape[1]] ).sum(1)/ test_preds.shape[0]
<compute_test_metric><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = 1 - test['target_final'].values
sub.to_csv('submission.csv',index=False)
 plt.hist(1 - test['target_final'].values ,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()