Kaggle competition: Tabular Playground Series - Feb 2021
%%time
for df in [df_train, df_test]:
df['sum'] = df[train_cols].sum(axis=1)
 df['min'] = df[train_cols].min(axis=1)
 df['max'] = df[train_cols].max(axis=1)
 df['mean'] = df[train_cols].mean(axis=1)
 df['std'] = df[train_cols].std(axis=1)
 df['skew'] = df[train_cols].skew(axis=1)
 df['kurt'] = df[train_cols].kurtosis(axis=1)
 df['med'] = df[train_cols].median(axis=1 )<feature_engineering>
import lightgbm as lgb
import optuna.integration.lightgbm as oplgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import category_encoders as ce
import seaborn as sns
%%time
for key in interactions:
for value in interactions[key]:
df_train[key+'_'+value+'_mul'] = df_train[key]*df_train[value]
df_train[key+'_'+value+'_div'] = df_train[key]/df_train[value]
df_train[key+'_'+value+'_sum'] = df_train[key] + df_train[value]
df_train[key+'_'+value+'_sub'] = df_train[key] - df_train[value]
df_test[key+'_'+value+'_mul'] = df_test[key]*df_test[value]
df_test[key+'_'+value+'_div'] = df_test[key]/df_test[value]
df_test[key+'_'+value+'_sum'] = df_test[key] + df_test[value]
df_test[key+'_'+value+'_sub'] = df_test[key] - df_test[value]<data_type_conversions>
df_train = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/train.csv")
 df_test = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/test.csv")
 df_sample = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/sample_submission.csv" )
df_train['num_zero_rows'] =(df_train_raw[train_cols] == 0 ).astype(int ).sum(axis=1)
 df_test['num_zero_rows'] =(df_test_raw[train_cols] == 0 ).astype(int ).sum(axis=1 )<define_variables>
train_id = df_train["id"]
test_id = df_test["id"]
df_train.drop("id", axis=1, inplace=True)
 df_test.drop("id", axis=1, inplace=True )
all_columns = [col for col in df_train.columns if col not in ['ID_code', 'target']]<init_hyperparams>
cat_features = [f"cat{i}" for i in range(9 + 1)]
params = {
'num_leaves': 13,
'max_bin': 63,
'min_data_in_leaf': 80,
'learning_rate': 0.0081,
'min_sum_hessian_in_leaf': 10.0,
'bagging_fraction': 0.331,
'bagging_freq': 5,
'max_depth': -1,
'save_binary': True,
'feature_fraction': 0.041,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'boost_from_average': False,
'device': 'gpu',
'gpu_platform_id':0,
'gpu_device_id': 0,
'seed':44000
}
num_round = 20000<split>
onehot_encoder = ce.one_hot.OneHotEncoder()
onehot_encoder.fit(pd.concat([df_train[cat_features], df_test[cat_features]], axis=0))
train_ohe = onehot_encoder.transform(df_train[cat_features])
 test_ohe = onehot_encoder.transform(df_test[cat_features])
 train_ohe.columns = [f"OHE_{col}" for col in train_ohe]
test_ohe.columns = [f"OHE_{col}" for col in test_ohe]
%%time
folds = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 44000)
 oof = np.zeros(len(df_train))
predictions = np.zeros(len(df_test))
feature_import_df = pd.DataFrame()
for n_fold,(train_idx, val_idx)in enumerate(folds.split(df_train, y_train)) :
print("fold number =", n_fold+1)
 train_data = lgb.Dataset(df_train.iloc[train_idx][all_columns], label = y_train.iloc[train_idx])
 val_y = y_train.iloc[val_idx]
val_data = lgb.Dataset(df_train.iloc[val_idx][all_columns], label = val_y)
 watchlist = [train_data,val_data]
clf = lgb.train(params, train_data, num_boost_round = num_round,
valid_sets = watchlist, verbose_eval = 4000,
early_stopping_rounds=3000)
 oof[val_idx] = clf.predict(df_train.iloc[val_idx][all_columns], num_iteration=clf.best_iteration)
 fold_import_df = pd.DataFrame()
fold_import_df['Feature'] = all_columns
fold_import_df["importance"] = clf.feature_importance()
fold_import_df['fold'] = n_fold +1
feature_import_df = pd.concat([feature_import_df,fold_import_df], axis = 0)
 predictions += clf.predict(df_test[all_columns])/folds.n_splits
print("\tFold AUC Score: {}\tf1_score: {}
".format(roc_auc_score(val_y,oof[val_idx]),
f1_score(val_y,np.round(oof[val_idx]))))
gc.collect()
print("
CV AUC Score and std", roc_auc_score(y_train, oof),np.std(oof))
print("CV F1 Score", f1_score(y_train, np.round(oof)) )<save_to_csv>
ordinal_encoder = ce.ordinal.OrdinalEncoder()
ordinal_encoder.fit(pd.concat([df_train[cat_features], df_test[cat_features]], axis=0))
train_or = ordinal_encoder.transform(df_train[cat_features])
 test_or = ordinal_encoder.transform(df_test[cat_features])
 train_or.columns = [f"OR_{col}" for col in train_or]
test_or.columns = [f"OR_{col}" for col in test_or]
sub = pd.DataFrame({'ID_code': df_test.ID_code.values,
'target': predictions})
 sub.to_csv('lgbm_0401_kernelgpu.csv', index = False )<set_options>
numerical_features = [f"cont{i}" for i in range(13 + 1)]
gc.enable()<load_from_csv>
train_x = pd.concat([
df_train[numerical_features],
train_ohe,
train_or
], axis=1 )
df_train = pd.read_csv('.. /input/train.csv' )<load_from_csv>
test_x = pd.concat([
df_test[numerical_features],
test_ohe,
test_or
], axis=1 )
df_test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
train_y = df_train["target"]
df_sub = pd.read_csv('.. /input/sample_submission.csv' )<sort_values>
oplgb_train_data = oplgb.Dataset(train_x, train_y )
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() /df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head()<feature_engineering>
oplgb_params = {
"objective": "regression",
"metric": "root_mean_squared_error",
"verbosity": -1,
"learning_rate": 0.01
}
def featureEngineering(df,cols,col_name):
df[col_name] = 0
for c in cols:
df[col_name] += df[c]
return df<compute_test_metric>
folds = KFold(n_splits=5, shuffle=True, random_state=2021 )
df_test = featureEngineering(df_test,cols, 'posCols' )<prepare_x_and_y>
tuner_cv = oplgb.LightGBMTunerCV(oplgb_params, oplgb_train_data, num_boost_round=1000, early_stopping_rounds=100, folds=folds, verbose_eval=100, time_budget=21600)
 tuner_cv.run()
val = np.zeros(df_train.shape[0])
 pred = np.zeros(df_test.shape[0])
 x = df_train.drop(['ID_code','target'],axis=1 ).values
y = df_train['target'].values
folds = StratifiedKFold(n_splits=3, shuffle=True, random_state=1234 )<choose_model_class>
tuner_cv.best_params
model_xgb = xgb.XGBClassifier(max_depth=2,
colsample_bytree=0.7,
n_estimators=20000,
scale_pos_weight = 9,
learning_rate=0.02,
objective='binary:logistic',
verbosity =1,
eval_metric = 'auc',
tree_method='gpu_hist',
n_jobs=-1 )<find_best_model_class>
class FoldsAverageLGBM:
def __init__(self, folds):
self.folds = folds
self.models = []
def fit(self, lgb_params, train_x, train_y):
oof_preds = np.zeros_like(train_y)
 self.train_x = train_x
self.train_y = train_y.values
for tr_idx, va_idx in tqdm(folds.split(train_x)) :
tr_x, va_x = self.train_x.iloc[tr_idx], self.train_x.iloc[va_idx]
tr_y, va_y = self.train_y[tr_idx], self.train_y[va_idx]
lgb_train_dataset = lgb.Dataset(tr_x, tr_y)
 lgb_valid_dataset = lgb.Dataset(va_x, va_y)
 model = lgb.train(lgb_params, lgb_train_dataset, valid_sets=[lgb_valid_dataset], verbose_eval=100)
 self.models.append(model)
 oof_pred = model.predict(va_x)
 oof_preds[va_idx] = oof_pred
self.oof_preds = oof_preds
def predict(self, test_x):
preds = []
for model in tqdm(self.models):
pred = model.predict(test_x)
 preds.append(pred)
 preds = np.mean(preds, axis=0)
 return preds
def get_feature_importance(self, importance_type="gain"):
feature_names = self.models[0].feature_name()
feature_importances_list = [model.feature_importance(importance_type)for model in self.models]
out_df = pd.DataFrame()
for i, name in enumerate(feature_names):
out_df[name] = [v[i] for v in feature_importances_list]
return out_df
for fold_index,(train_index,val_index)in enumerate(folds.split(x,y)) :
print('Batch {} started...'.format(fold_index))
gc.collect()
bst = model_xgb.fit(x[train_index],y[train_index],
eval_set = [(x[val_index],y[val_index])],
early_stopping_rounds=200,
verbose= 200,
eval_metric ='auc' 
)
 val[val_index] = model_xgb.predict_proba(x[val_index])[:,1]
print('auc of this val set is {}'.format(roc_auc_score(y[val_index],val[val_index])))
 pred += model_xgb.predict_proba(df_test.drop(['ID_code'],axis=1 ).values)[:,1]/folds.n_splits<prepare_output>
lgb_params = dict(tuner_cv.best_params)
 lgb_params["learning_rate"] = 0.001
lgb_params["early_stopping_round"] = 1000
lgb_params["num_iterations"] = 20000
df_sub.target = pred<save_to_csv>
folds_average_lgbm = FoldsAverageLGBM(folds )
df_sub.to_csv('r_prob.csv',index=False )<load_from_csv>
folds_average_lgbm.fit(lgb_params, train_x, train_y )
train = pd.read_csv('.. /input/train.csv')
 train0 = train[ train['target']==0 ].copy()
train1 = train[ train['target']==1 ].copy()
train.sample(5 )<feature_engineering>
np.sqrt(mean_squared_error(df_train.target, folds_average_lgbm.oof_preds))
s = [0]*200
m = [0]*200
for i in range(200):
s[i] = np.std(train['var_'+str(i)])
 m[i] = np.mean(train['var_'+str(i)])
 def getp(i,x):
c = 3
a = len(train1[(train1['var_'+str(i)]>x-s[i]/c)&(train1['var_'+str(i)]<x+s[i]/c)])
 b = len(train0[(train0['var_'+str(i)]>x-s[i]/c)&(train0['var_'+str(i)]<x+s[i]/c)])
 if a+b<500: return 0.1
return a /(a+b)
 def smooth(x,st=1):
for j in range(st):
x2 = np.ones(len(x)) * 0.1
for i in range(len(x)-2):
x2[i+1] = 0.25*x[i]+0.5*x[i+1]+0.25*x[i+2]
x = x2.copy()
return x<define_variables>
y_pred = folds_average_lgbm.predict(test_x )
<compute_test_metric><EOS> 
sub = df_sample.copy()
sub["target"] = y_pred
sub.to_csv("submission_optuna_lgbm_ohe_or_v1.csv", index=False)
 sub.head()