Kaggle competition: House Prices - Advanced Regression Techniques
features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target = train_df['target']<split>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
%%time
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2019)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
start = time.time()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ 5
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("lgb_submission.csv", index=False )<split>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
model = CatBoostClassifier(loss_function="Logloss", eval_metric="AUC")
 kf = KFold(n_splits=5, random_state=42, shuffle=True)
 y_valid_pred = 0 * target
y_test_pred = 0
for idx,(train_index, valid_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = target.iloc[train_index], target.iloc[valid_index]
X_train, X_valid = train_df[features].iloc[train_index,:], train_df[features].iloc[valid_index,:]
_train = Pool(X_train, label=y_train)
 _valid = Pool(X_valid, label=y_valid)
 print("
Fold ", idx)
 fit_model = model.fit(_train,
eval_set=_valid,
use_best_model=True,
verbose=200 
)
 pred = fit_model.predict_proba(X_valid)[:,1]
print(" auc = ", roc_auc_score(y_valid, pred))
y_valid_pred.iloc[valid_index] = pred
y_test_pred += fit_model.predict_proba(test_df[features])[:,1]
y_test_pred /= 5<save_to_csv>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
sub_df1 = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df1["target"] = y_test_pred
sub_df1.to_csv("cat_submission.csv", index=False )<merge>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
corr_df = pd.merge(sub_df,sub_df1,how='left',on='ID_code')
 corr_df.corr()<save_to_csv>
all_data["Alley"] = all_data["Alley"].fillna("None" )
sub_df2 = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df2["target"] = 0.5*sub_df["target"] + 0.5*sub_df1["target"]
sub_df2.to_csv("lgb_cat_submission.csv", index=False )<set_options>
all_data["Fence"] = all_data["Fence"].fillna("None" )
pd.set_option('display.max_columns', 200 )<load_from_csv>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<count_values>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
train_df.target.value_counts()<init_hyperparams>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
param = {
'num_leaves': 18,
'max_bin': 63,
'min_data_in_leaf': 5,
'learning_rate': 0.010614430970330217,
'min_sum_hessian_in_leaf': 0.0093586657313989123,
'feature_fraction': 0.056701788569420042,
'lambda_l1': 0.060222413158420585,
'lambda_l2': 4.6580550589317573,
'min_gain_to_split': 0.29588543202055562,
'max_depth': 49,
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False
} 
<define_variables>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
nfold = 10<define_variables>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
target = 'target'
predictors = train_df.columns.values.tolist() [2:]<set_options>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
gc.collect()<split>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=2019)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 nround = 8523
clf = lgb.train(param, xg_train, nround, valid_sets = [xg_valid], verbose_eval=250)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=nround)
 predictions += clf.predict(test_df[predictors], num_iteration=nround)/ nfold
i = i + 1
print("
CV AUC: {:<0.4f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )<prepare_output>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = predictions
sub_df[:10]<save_to_csv>
all_data = all_data.drop(['Utilities'], axis=1 )
sub_df.to_csv("Customer_Transaction.csv", index=False )<load_from_csv>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )<define_variables>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
<categorify>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
melted1 = pd.melt(train.iloc[:, np.r_[1, 2:42]], id_vars='target')
 melted2 = pd.melt(train.iloc[:, np.r_[1, 42:82]], id_vars='target')
 melted3 = pd.melt(train.iloc[:, np.r_[1, 82:122]], id_vars='target')
 melted4 = pd.melt(train.iloc[:, np.r_[1, 122:162]], id_vars='target')
 melted5 = pd.melt(train.iloc[:, np.r_[1, 162:202]], id_vars='target' )<train_on_grid>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
logit_mod = sm.Logit(train['target'], train.iloc[:, 2:])
 logit_res = logit_mod.fit(disp=0)
 print(logit_res.summary() )<import_modules>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
import lightgbm as lgb
from sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix
from sklearn.model_selection import StratifiedKFold<init_hyperparams>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
param = {
'num_leaves': 5,
'max_depth': 15,
'save_binary': True,
'seed': 42,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
}<split>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
train_preds = np.zeros(len(train))
test_preds = np.zeros(len(test))
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for train_index, valid_index in skf.split(train[variables], train['target']):
train_data = lgb.Dataset(train.loc[train_index, variables],
label=train.loc[train_index, 'target'])
 valid_data = lgb.Dataset(train.loc[valid_index, variables],
label=train.loc[valid_index, 'target'])
 bst = lgb.train(param, train_data, num_boost_round=2000, valid_sets=valid_data,
verbose_eval=500, early_stopping_rounds=30)
 train_preds[valid_index] = bst.predict(train.loc[valid_index, variables],
num_iteration=bst.best_iteration)
 test_preds += bst.predict(test[variables], num_iteration=bst.best_iteration)/ 5
print('Accuracy {}'.format(accuracy_score(train['target'], np.where(train_preds > 0.5, 1, 0))))
print('ROC AUC Score: {}'.format(roc_auc_score(train['target'], train_preds)) )<save_to_csv>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
submission = pd.DataFrame({'ID_code': test['ID_code'],
'target': test_preds})
 submission.to_csv('submission.csv', index=False )<import_modules>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
import numpy as np
import pandas as pd
import pydicom
import os
import sys
import matplotlib.pyplot as plt
import collections
from tqdm import tqdm_notebook as tqdm
from datetime import datetime
from math import ceil, floor
import cv2
import tensorflow as tf
import keras
import seaborn as sns
from glob import glob
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.optimizers import Optimizer
from keras.callbacks import *<define_variables>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
BASE_PATH = '/kaggle/input/rsna_noben_png_224_224_v1/'
TRAIN_PATH = 'stage_1_train_images/'
TEST_PATH = 'stage_1_test_images/'
TMP_DIR_PATH = '/kaggle/tmp/'
TRAIN_TMP_DIR_PATH = TMP_DIR_PATH + 'train_png_img/'
TEST_TMP_DIR_PATH = TMP_DIR_PATH + 'test_png_img/'
IMG_SIZE = 224<load_from_csv>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
train_df_csv = pd.read_csv(BASE_PATH + 'stage_1_train.csv')
 print(train_df_csv.shape)
<feature_engineering>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
train_df_csv['filename'] = train_df_csv['ID'].apply(lambda st: "ID_" + st.split('_')[1] + ".dcm")
 train_df_csv['type'] = train_df_csv['ID'].apply(lambda st: st.split('_')[2])
 print(train_df_csv.shape)
<count_unique_values>
train = all_data[:ntrain]
test = all_data[ntrain:]
train_df_csv.filename.nunique()<count_values>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
label_dist = train_df_csv.groupby("type" ).Label.value_counts().unstack().reset_index()
label_dist<concatenate>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
pivot_train_df_csv=pd.concat([pivot_train_df_csv_any,pivot_train_df_csv_nany] )<load_from_csv>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
pivot_train_df_csv=pivot_train_df_csv.reset_index(drop=True )<feature_engineering>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
test_df_csv = pd.read_csv(BASE_PATH + 'stage_1_sample_submission.csv')
 test_df_csv['filename'] = test_df_csv['ID'].apply(lambda st: "ID_" + st.split('_')[1] + ".dcm")
 test_df_csv['type'] = test_df_csv['ID'].apply(lambda st: st.split('_')[2])
 print(test_df_csv.shape)
 test_df_csv.head()<create_dataframe>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
test_df_csv = pd.DataFrame(test_df_csv.filename.unique() , columns=['filename'])
 print(test_df_csv.shape)
<feature_engineering>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
pivot_train_df_csv['filename'] = pivot_train_df_csv['filename'].apply(lambda st: st.split('.')[0] + ".png")
 pivot_train_df_csv.head(2 )<feature_engineering>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
test_df_csv['filename'] = test_df_csv['filename'].apply(lambda st: st.split('.')[0] + ".png")
 test_df_csv.head(2 )<define_variables>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
print(BASE_PATH + TEST_PATH)
 train_dcm_img = sorted(glob(BASE_PATH + TRAIN_PATH + "*.png"))
test_dcm_img = sorted(glob(BASE_PATH + TEST_PATH + "*.png"))
print("train files: ", len(train_dcm_img))
print("test files: ", len(test_dcm_img))
pd.reset_option('max_colwidth' )<define_variables>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
num_ill_patients = pivot_train_df_csv[pivot_train_df_csv["any"]==1].shape[0]
num_ill_patients<randomize_order>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
undersample_seed = 42
healthy_patients = pivot_train_df_csv[pivot_train_df_csv["any"]==0].index.values
healthy_patients_selection = np.random.RandomState(undersample_seed ).choice(
 healthy_patients, size=num_ill_patients, replace=False 
)
 len(healthy_patients_selection )<init_hyperparams>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def create_datagen() :
train_aug = ImageDataGenerator(rescale=1./255,
horizontal_flip = True,
rotation_range = 10,
validation_split=0.20,
preprocessing_function=preprocess_image 
)
 return train_aug<define_variables>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
BATCH_SIZE = 32
def create_test_gen() :
return ImageDataGenerator(rescale=1./255,
preprocessing_function=preprocess_image ).flow_from_dataframe(
 test_df_csv,
directory= BASE_PATH+TEST_PATH,
x_col='filename',
class_mode=None,
target_size=(IMG_SIZE, IMG_SIZE),
batch_size=BATCH_SIZE,
shuffle=False
)<prepare_x_and_y>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def create_flow(datagen, subset):
return datagen.flow_from_dataframe(
 pivot_train_df_csv,
directory =BASE_PATH+TRAIN_PATH,
x_col = 'filename',
y_col = ['any', 'epidural', 'intraparenchymal','intraventricular', 'subarachnoid', 'subdural'],
class_mode='raw',
target_size=(IMG_SIZE, IMG_SIZE),
batch_size=BATCH_SIZE,
subset=subset
)<define_variables>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
%%time
train_data_generator = create_datagen()
train_gen = create_flow(train_data_generator, 'training')
 val_gen = create_flow(train_data_generator, 'validation')
 test_gen = create_test_gen()<import_modules>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
from keras.applications.xception import Xception
from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D, Dropout, Input
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from sklearn.model_selection import train_test_split
import keras
from keras.engine import Layer,InputSpec
from keras.applications.resnet50 import ResNet50<choose_model_class>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<compute_train_metric>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
def custom_log_loss(y_true, y_pred):
loss_val = tf.losses.log_loss(y_true, y_pred)
 return loss_val<choose_model_class>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
clr = CyclicLR(base_lr=0.001, max_lr=0.009,
step_size=300., mode='exp_range',
gamma=0.99994 )<load_pretrained>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
input_layer = Input(shape =(IMG_SIZE,IMG_SIZE,3))
base_model = ResNet50(weights = None,
include_top = False,
input_tensor = input_layer)
 base_model.load_weights('.. /input/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(512, activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.3 )(x)
 out = Dense(6, activation = 'sigmoid' )(x)
 model = Model(inputs = input_layer, outputs = out)
 model.summary()<choose_model_class>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
filepath = "resNet50_DS_5epoch.h5"
mc=ModelCheckpoint(filepath, monitor='custom_log_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='min')
 callback_list = [clr,mc]
model.compile(optimizer = optimizer, loss = custom_log_loss, metrics = ["accuracy",custom_log_loss])
 model.summary() 
<define_variables>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
class_weights = {0: 2.,
1: 1.,
2: 1.,
3: 1.,
4: 1.,
5: 1.}<train_model>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
%%time
history = model.fit_generator(generator = train_gen,
steps_per_epoch = len(train_gen),
epochs = 5,
validation_data = val_gen,
validation_steps = len(val_gen),
callbacks = callback_list,
class_weight=class_weights )<load_pretrained>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
model.save_weights("Weight_resNet50_DS_5epoch_2.h5")
 model.save('resNet50_DS_5epoch_2.h5' )<predict_on_test>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
y_test = model.predict_generator(
 test_gen,
steps=len(test_gen),
verbose=1
)<save_to_csv>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )