Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<init_hyperparams>
%matplotlib inline
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : 4,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}<create_dataframe>
!pip install --upgrade xgboost
xgb.__version__
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()<prepare_x_and_y>
shap.initjs()
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values<split>
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-feb-2021/sample_submission.csv')
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, df_train['target'])) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
N = 5
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N<groupby>
target = train['target'].values
print("Distribution of 1s in original data : {} / {} ".format(np.sum(y_train), len(y_train)))
 print("Percentage of 1s in original data : {}".format(np.sum(y_train)*100.0/len(y_train)))
 print("Percentage of 1s in augmented data : {}".format(np.sum(y_t)*100.0/len(y_t)))
 print("Distribution of 1s in augmented data : {} / {} ".format(np.sum(y_t), len(y_t)))
<compute_test_metric>
for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature] )
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))<save_to_csv>
train_oof = np.zeros(( 300000,))
test_preds = 0
train_oof.shape
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )<set_options>
xgb_params= {
"objective": "reg:squarederror",
"max_depth": 6,
"learning_rate": 0.01,
"colsample_bytree": 0.4,
"subsample": 0.6,
"reg_alpha" : 6,
"min_child_weight": 100,
"n_jobs": 2,
"seed": 2001,
'tree_method': "gpu_hist",
"gpu_id": 0,
'predictor': 'gpu_predictor'
}
warnings.filterwarnings('ignore')
 mpl.style.use('seaborn-notebook')
 sns.set_style('whitegrid' )<load_from_csv>
test = xgb.DMatrix(test[columns] )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape, test.shape<feature_engineering>
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(xgb_params, train_df, 3800)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(test)
 train_oof[val_ind] = temp_oof
test_preds += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
def missing(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
mean_squared_error(train_oof, target, squared=False)
missing(train )<count_missing_values>
np.save('train_oof', train_oof)
 np.save('test_preds', test_preds )
missing(test )<count_values>
%%time
shap_preds = model.predict(test, pred_contribs=True )
print(' {} % of 1 are there in Training
{} % of 0 are there in Training'.format(((train['target'].value_counts() [1]/train.shape[0])*100),(( train['target'].value_counts() [0]/train.shape[0])*100)) )<count_unique_values>
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature] )
features = train.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train[feature].value_counts()
unique_max_train.append([feature, values.max() , values.idxmax() ])
 values = test[feature].value_counts()
unique_max_test.append([feature, values.max() , values.idxmax() ] )<feature_engineering>
%%time
shap_interactions = model.predict(xgb.DMatrix(test[columns]), pred_interactions=True )
idx = features = train.columns.values[2:202]
for df in [test, train]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )<feature_engineering>
train['cont8_cont0'] = train['cont8']*train['cont0']
train['cont9_cont0'] = train['cont9']*train['cont0']
train['cont9_cont5'] = train['cont9']*train['cont5']
train['cont8_cont5'] = train['cont8']*train['cont5']
test['cont8_cont0'] = test['cont8']*test['cont0']
test['cont9_cont0'] = test['cont9']*test['cont0']
test['cont9_cont5'] = test['cont9']*test['cont5']
test['cont8_cont5'] = test['cont8']*test['cont5']
features = [c for c in train.columns if c not in ['ID_code', 'target']]
for feature in features:
train['r2_'+feature] = np.round(train[feature], 2)
 test['r2_'+feature] = np.round(test[feature], 2)
 train['r1_'+feature] = np.round(train[feature], 1)
 test['r1_'+feature] = np.round(test[feature], 1 )<init_hyperparams>
del shap_interactions, shap_preds
gc.collect()
gc.collect()
features = [c for c in train.columns if c not in ['ID_code', 'target']]
target = train['target']
param = {
'bagging_freq': 5,
'bagging_fraction': 0.4,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.05,
'learning_rate': 0.01,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<train_model>
test = xgb.DMatrix(test[columns] )
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=44000)
 oof = np.zeros(len(train))
predictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
xgb_params= {
"objective": "reg:squarederror",
"max_depth": 6,
"learning_rate": 0.01,
"colsample_bytree": 0.4,
"subsample": 0.6,
"reg_alpha" : 6,
"min_child_weight": 100,
"n_jobs": 2,
"seed": 2001,
'tree_method': "gpu_hist",
"gpu_id": 0,
'predictor': 'gpu_predictor'
}
sub = pd.DataFrame({"ID_code":test["ID_code"].values})
 sub["target"] = predictions
sub.to_csv("submission.csv", index=False )<install_modules>
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 for f,(train_ind, val_ind)in tqdm(enumerate(kf.split(train, target))):
train_df, val_df = train.iloc[train_ind][columns], train.iloc[val_ind][columns]
train_target, val_target = target[train_ind], target[val_ind]
train_df = xgb.DMatrix(train_df, label=train_target)
 val_df = xgb.DMatrix(val_df, label=val_target)
 model = xgb.train(xgb_params, train_df, 3800)
 temp_oof = model.predict(val_df)
 temp_test = model.predict(test)
 train_oof_2[val_ind] = temp_oof
test_preds_2 += temp_test/NUM_FOLDS
print(mean_squared_error(temp_oof, val_target, squared=False))
!rm -r /opt/conda/lib/python3.6/site-packages/lightgbm
!git clone --recursive https://github.com/Microsoft/LightGBM<install_modules>
mean_squared_error(train_oof_2, target, squared=False)
!apt-get install -y -qq libboost-all-dev<install_modules>
mean_squared_error(0.5*train_oof+0.5*train_oof_2, target, squared=False)
%%bash
cd LightGBM
rm -r build
mkdir build
cd build
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/..
make -j$(nproc )<install_modules>
np.save('train_oof_2', train_oof_2)
 np.save('test_preds_2', test_preds_2 )
!cd LightGBM/python-package/;python3 setup.py install --precompile<set_options>
%%time
shap_preds = model.predict(test, pred_contribs=True )
!nvidia-smi<set_options>
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv')
 for feature in cat_features:
le = LabelEncoder()
le.fit(train[feature])
 train[feature] = le.transform(train[feature])
 test[feature] = le.transform(test[feature])
 train['cont8_cont0'] = train['cont8']*train['cont0']
train['cont9_cont0'] = train['cont9']*train['cont0']
train['cont9_cont5'] = train['cont9']*train['cont5']
train['cont8_cont5'] = train['cont8']*train['cont5']
test['cont8_cont0'] = test['cont8']*test['cont0']
test['cont9_cont0'] = test['cont9']*test['cont0']
test['cont9_cont5'] = test['cont9']*test['cont5']
test['cont8_cont5'] = test['cont8']*test['cont5']
columns = test.columns[1:]
%matplotlib inline
pd.set_option('display.max_columns', 1000)
 warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 pd.set_option('display.max_rows', 1000)
 pd.set_option('display.max_columns', 1000)
<load_from_csv>
sub['target'] = test_preds
sub.to_csv('submission.csv', index=False )
%%time
df_train_raw = pd.read_csv('.. /input/train.csv')
 df_test_raw = pd.read_csv('.. /input/test.csv' )<create_dataframe>
sub['target'] = test_preds_2
sub.to_csv('submission_2.csv', index=False )
%%time
df_train = df_train_raw.copy()
df_test = df_test_raw.copy()<prepare_x_and_y>
sub['target'] = 1.1*test_preds-0.1*test_preds_2
sub.to_csv('submission_average_0.csv', index=False )
train_cols = [col for col in df_train.columns if col not in ['ID_code', 'target']]
y_train = df_train['target']<define_variables>
sub['target'] = 1.2*test_preds-0.2*test_preds_2
sub.to_csv('submission_average_1.csv', index=False )
<feature_engineering><EOS> 
sub['target'] = 1.3*test_preds-0.3*test_preds_2
sub.to_csv('submission_average_2.csv', index=False )