Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<choose_model_class>
warnings.filterwarnings('ignore' )
class CyclicLR(Callback):

 def __init__(
 self,
base_lr=0.001,
max_lr=0.006,
step_size=2000.,
mode='triangular',
gamma=1.,
scale_fn=None,
scale_mode='cycle'):
super(CyclicLR, self ).__init__()
if mode not in ['triangular', 'triangular2',
'exp_range']:
raise KeyError("mode must be one of 'triangular', "
"'triangular2', or 'exp_range'")
 self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma ** x
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr is not None:
self.base_lr = new_base_lr
if new_max_lr is not None:
self.max_lr = new_max_lr
if new_step_size is not None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* \
np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* \
np.maximum(0,(1 - x)) * self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr())
 self.history.setdefault(
 'lr', [] ).append(
 K.get_value(
 self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
logs['lr'] = K.get_value(self.model.optimizer.lr )<load_from_csv>
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv' )
train_df = pd.read_csv('.. /input/train.csv', index_col=0)
 test_df = pd.read_csv('.. /input/test.csv', index_col=0 )<prepare_x_and_y>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train.target
X_test = test.drop(['id'], axis=1 )
X = train_df.drop(['target'],axis=1)
 y = train_df['target']<prepare_x_and_y>
cat_cols = [feature for feature in train.columns if 'cat' in feature]
def label_encoder(df):
for feature in cat_cols:
le = LabelEncoder()
le.fit(df[feature])
 df[feature] = le.transform(df[feature])
 return df
X_test = test_df<normalization>
X_train = label_encoder(X_train)
 X_test = label_encoder(X_test )
scaler = StandardScaler()
X_s=scaler.fit_transform(X)
 X_test_s=scaler.transform(X_test )<create_dataframe>
split = KFold(n_splits=5, random_state=2 )
X=pd.DataFrame(X_s)
 X_test=pd.DataFrame(X_test_s )<feature_engineering>
def objective(trial, X, y, name='xgb'):
params = {'max_depth':trial.suggest_int('max_depth', 5, 50),
'n_estimators':200000,
'subsample': trial.suggest_uniform('subsample', 0.2, 1.0),
'colsample_bytree':trial.suggest_uniform('colsample_bytree', 0.2, 1.0),
'learning_rate':trial.suggest_uniform('learning_rate', 0.007, 0.02),
'reg_lambda':trial.suggest_uniform('reg_lambda', 0.01, 50),
'reg_alpha':trial.suggest_uniform('reg_alpha', 0.01, 50),
'min_child_samples':trial.suggest_int('min_child_samples', 5, 100),
'num_leaves':trial.suggest_int('num_leaves', 10, 200),
'n_jobs' : -1,
'metric':'rmse',
'max_bin':trial.suggest_int('max_bin', 300, 1000),
'cat_smooth':trial.suggest_int('cat_smooth', 5, 100),
'cat_l2':trial.suggest_loguniform('cat_l2', 1e-3, 100)}
model = LGBMRegressor(**params)
 X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=0)
 model.fit(X_train, y_train, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=250,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0)
 train_score = np.round(np.sqrt(mean_squared_error(y_train, model.predict(X_train))), 5)
 test_score = np.round(np.sqrt(mean_squared_error(y_val, model.predict(X_val))), 5)
 print(f'TRAIN RMSE : {train_score} || TEST RMSE : {test_score}')
 return test_score
X_test.index=test_df.index<define_variables>
optimize = partial(objective, X=X_train, y=y_train)
 study_lgbm = optuna.create_study(direction='minimize')
seed = 7
np.random.seed(seed )<choose_model_class>
lgbm_params = {'max_depth': 16,
'subsample': 0.8032697250789377,
'colsample_bytree': 0.21067140508531404,
'learning_rate': 0.009867383057779643,
'reg_lambda': 10.987474846877767,
'reg_alpha': 17.335285595031994,
'min_child_samples': 31,
'num_leaves': 66,
'max_bin': 522,
'cat_smooth': 81,
'cat_l2': 0.029690334194270022,
'metric': 'rmse',
'n_jobs': -1,
'n_estimators': 20000}
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=7 )<choose_model_class>
preds_list_base = []
preds_list_final_iteration = []
preds_list_all = []
for train_idx, val_idx in split.split(X_train):
X_tr = X_train.iloc[train_idx]
X_val = X_train.iloc[val_idx]
y_tr = y_train.iloc[train_idx]
y_val = y_train.iloc[val_idx]
Model = LGBMRegressor(**lgbm_params ).fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=250,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0)
 preds_list_base.append(Model.predict(X_test))
preds_list_all.append(Model.predict(X_test))
print(f'RMSE for Base model is {np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))}')
 first_rmse = np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))
 params = lgbm_params.copy()
for i in range(1, 8):
if i >2:
params['reg_lambda'] *= 0.9
params['reg_alpha'] *= 0.9
params['num_leaves'] += 40
params['learning_rate'] = 0.003
Model = LGBMRegressor(**params ).fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=200,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0,
init_model=Model)
 preds_list_all.append(Model.predict(X_test))
print(f'RMSE for Incremental trial {i} model is {np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))}')
 last_rmse = np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))
 print('',end='
')
 print(f'Improvement of : {first_rmse - last_rmse}')
 print('-' * 100)
 preds_list_final_iteration.append(Model.predict(X_test))
adam = optimizers.adam
model = Sequential()
model.add(Dense(64, input_shape=(200,1),
kernel_initializer='normal',
activation="relu"))
model.add(Flatten())
 model.add(Dropout(0.2))
model.add(Dense(1,activation="sigmoid"))
model.compile(loss="binary_crossentropy", optimizer='adam',metrics=['accuracy'])
 clr = CyclicLR(base_lr=0.001, max_lr=0.006,step_size=400., mode='triangular')
<train_model>
y_preds_base = np.array(preds_list_base ).mean(axis=0)
 y_preds_base
preds = []
c = 0
oof_preds = np.zeros(( len(X), 1))
for train, valid in cv.split(X, y):
print("VAL %s" % c)
 X_train = np.reshape(X.iloc[train].values,(-1, 200, 1))
y_train_ = y.iloc[train].values
X_valid = np.reshape(X.iloc[valid].values,(-1, 200, 1))
y_valid = y.iloc[valid].values
early = EarlyStopping(monitor="val_loss",
mode="min",
patience=20)
 model.fit(X_train, y_train_, validation_data=(X_valid, y_valid), epochs=200, verbose=2, batch_size=1024,
callbacks=[early,clr])
 X_test1 = np.reshape(X_test.values,(200000, 200, 1))
curr_preds = model.predict(X_test1, batch_size=1024)
 oof_preds[valid] = model.predict(X_valid)
 preds.append(curr_preds)
 c += 1
auc = roc_auc_score(y, oof_preds)
 print("CV_AUC: {}".format(auc)) 
<save_to_csv>
y_preds_all = np.array(preds_list_all ).mean(axis=0)
 y_preds_all
preds = np.asarray(preds)
 preds = preds.reshape(( 5, 200000))
preds_final = np.mean(preds.T, axis=1)
 submission = pd.read_csv('./.. /input/sample_submission.csv')
 submission['target'] = preds_final
submission.to_csv('submission.csv', index=False )<load_from_csv>
y_preds_final_iteration = np.array(preds_list_final_iteration ).mean(axis=0)
 y_preds_final_iteration
s1=pd.read_csv('.. /input/s1v8-nanashi-90-lines-solution-0901-fast/s1-v8.csv')['target']
s2=pd.read_csv('.. /input/santander-magic-lgb-0-901/submission.csv')['target']
s3=pd.read_csv('.. /input/s3v32-ashish-gupta-eda-pca-scaler-lgbm/s3-v32.csv')['target']
s4=pd.read_csv('.. /input/eda-pca-simple-lgbm-on-kfold-technique/submission26.csv')['target']
s5=pd.read_csv('.. /input/lgb-2-leaves-augment/lgb_submission.csv')['target']
s6=pd.read_csv('.. /input/s6v1-ole-morten-light-gbm-with-data-augment/s6-v1.csv')['target']
s7=pd.read_csv('.. /input/s7v19-subham-sharma-what-is-next-in-santander/s7-v19.csv')['target']
s8=pd.read_csv('.. /input/s8v5-joshua-reed-santander-customer-transaction/s8-v5.csv')['target']
s9=pd.read_csv('.. /input/s9v16-gauravtambi-lgbm-augmentation/s9-v16.csv')['target']
s10=pd.read_csv('.. /input/best-parameters-lb-0-900/submission.csv')['target']
submission = pd.read_csv('.. /input/santander-customer-transaction-prediction/sample_submission.csv')
 solutions_set = pd.DataFrame({'s1': s1, 's2': s2, 's3': s3, 's4': s4, 's5': s5, 's6': s6,
's7': s7, 's8': s8, 's9': s9, 's10': s10} )<normalization>
submission = pd.DataFrame({'id':test.id,
'target':y_preds_final_iteration} )
scaler = StandardScaler()
solutions_set_scaled = pd.DataFrame(scaler.fit_transform(solutions_set),
columns = ['s1', 's2', 's3', 's4', 's5', 's6',
's7', 's8', 's9', 's10'] )<normalization>
submission.to_csv('submission.csv', index=False )
pca = PCA(n_components = 1)
 factor = pca.fit_transform(solutions_set_scaled)
 print(pca.explained_variance_ratio_ )<save_to_csv>
submission.to_csv('submission.csv', index=False )
submission['target'] = solutions_set_scaled['factor']
filename="blended_submission_{:%Y-%m-%d_%H_%M}.csv".format(datetime.now())
 submission.to_csv(filename, index=False )<set_options>
pd.read_csv('submission.csv' )
%matplotlib inline 
<load_from_csv>
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=0)
 lgbm = LGBMRegressor(**lgbm_params ).fit(X_tr, y_tr, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=250,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=0 )
train = pd.read_csv('.. /input/train.csv', index_col=0)
 test = pd.read_csv('.. /input/test.csv', index_col=0)
 target = train.target.values
train.drop('target', axis=1, inplace=True)
 train.shape, target.shape, test.shape,<prepare_x_and_y>
def objective(trial, model, X, y, iterations=5):
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=0)
 f1 = trial.suggest_uniform('f1', 0.1, 1.0)
 f2 = trial.suggest_uniform('f2', 0.1, 3)
 f3 = trial.suggest_int('f3', 20, 100)
 f4 = trial.suggest_int('f4', 20, 50)
 f5 = trial.suggest_int('f5', 1, 5)
 lr_factor = trial.suggest_uniform('lr_factor', 0.1, 0.7)
 params = lgbm_params.copy()
print(f'RMSE for base model is {np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))}')
 for i in range(1, iterations):
if i > 2:
params['reg_lambda'] *= f1
params['reg_alpha'] += f2
params['num_leaves'] += f3
params['min_child_samples'] -= f4
params['cat_smooth'] -= f5
params['learning_rate'] *= lr_factor
params['learning_rate'] = params['learning_rate'] if params['learning_rate'] > 0.0009 else 0.0009
Model = model(**params ).fit(X_train, y_train, eval_set=[(X_val, y_val)],
eval_metric=['rmse'],
early_stopping_rounds=200,
categorical_feature=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
verbose=1000,
init_model=Model if i > 1 else lgbm)
 print(f'RMSE for {i}th model is {np.sqrt(mean_squared_error(y_val, Model.predict(X_val)))}')
 RMSE = mean_squared_error(y_val, Model.predict(X_val), squared=False)
 return RMSE
<prepare_x_and_y><EOS> 
study = optuna.create_study(direction='minimize')
 optimize = partial(objective, X=X_train, y=y_train, model=LGBMRegressor)