Kaggle competition: Tabular Playground Series - Jan 2021
pd.set_option('display.max_columns', None )<merge>
train_df = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv", index_col=["id"])
 test_df = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 X = train_df.iloc[:, :-1].to_numpy()
y = train_df.iloc[:, -1].to_numpy()
X_test = test_df.to_numpy()
final_train = pd.merge(train_transaction, train_identity, how = 'left', on = 'TransactionID')
 final_test = pd.merge(test_transaction, test_identity, how = 'left', on = 'TransactionID')
<drop_column>
def train(model):
X_train, X_test, y_train, y_test = train_test_split(X, y.flatten() , test_size=0.1, random_state=156)
 y_train = y_train.reshape(-1, 1)
 y_test = y_test.reshape(-1, 1)
 model = model.fit(X_train, y_train, early_stopping_rounds=100, verbose=False, eval_set=[(X_test, y_test)])
 score = mean_squared_error(model.predict(X_train), y_train, squared=False)
 print(score)
 return model
del train_transaction
del train_identity
del test_transaction
del test_identity<rename_columns>
def objectiveXGB(trial: Trial, X, y, test):
param = {
"n_estimators" : trial.suggest_int('n_estimators', 500, 4000),
'max_depth':trial.suggest_int('max_depth', 8, 16),
'min_child_weight':trial.suggest_int('min_child_weight', 1, 300),
'gamma':trial.suggest_int('gamma', 1, 3),
'learning_rate': 0.01,
'colsample_bytree':trial.suggest_discrete_uniform('colsample_bytree',0.5, 1, 0.1),
'nthread' : -1,
'tree_method': 'gpu_hist',
'predictor': 'gpu_predictor',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'subsample': trial.suggest_categorical('subsample', [0.6,0.7,0.8,1.0]),
'random_state': 42
}
X_train, X_test, y_train, y_test = train_test_split(X, y.flatten() , test_size=0.1)
 y_train = y_train.reshape(-1, 1)
 y_test = y_test.reshape(-1, 1)
 model = xgb.XGBRegressor(**param)
 xgb_model = model.fit(X_train, y_train, verbose=False, eval_set=[(X_test, y_test)])
 score = mean_squared_error(xgb_model.predict(X_test), y_test, squared=False)
 return score
final_test = final_test.rename(columns = {"id-01": "id_01", "id-02": "id_02", "id-03": "id_03",
"id-06": "id_06", "id-05": "id_05", "id-04": "id_04",
"id-07": "id_07", "id-08": "id_08", "id-09": "id_09",
"id-10": "id_10", "id-11": "id_11", "id-12": "id_12",
"id-15": "id_15", "id-14": "id_14", "id-13": "id_13",
"id-16": "id_16", "id-17": "id_17", "id-18": "id_18",
"id-21": "id_21", "id-20": "id_20", "id-19": "id_19",
"id-22": "id_22", "id-23": "id_23", "id-24": "id_24",
"id-27": "id_27", "id-26": "id_26", "id-25": "id_25",
"id-28": "id_28", "id-29": "id_29", "id-30": "id_30",
"id-31": "id_31", "id-32": "id_32", "id-33": "id_33",
"id-34": "id_34", "id-35": "id_35", "id-36": "id_36",
"id-37": "id_37", "id-38": "id_38"})
 final_test.head()<count_missing_values>
study = optuna.create_study(direction='minimize',sampler=TPESampler())
 study.optimize(lambda trial : objectiveXGB(trial, X, y, X_test), n_trials=50)
 print('Best trial: score {},
params {}'.format(study.best_trial.value,study.best_trial.params))
best_param = study.best_trial.params
xgbReg = train(xgb.XGBRegressor(**best_param, tree_method='gpu_hist', random_state=42, predictor='gpu_predictor', learning_rate=0.01, nthread=-1))
def getNulls(data):
total = data.isnull().sum()
percent = data.isnull().sum() / data.isnull().count()
missing_data = pd.concat([total, percent], axis = 1, keys = ['total', 'precent'])
 return missing_data<count_missing_values>
def objectiveLGBM(trial: Trial, X, y, test):
param = {
'objective': 'regression',
'metric': 'root_mean_squared_error',
'verbosity': -1,
'boosting_type': 'gbdt',
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'num_leaves': trial.suggest_int('num_leaves', 2, 512),
'learning_rate': 0.01,
'n_estimators': trial.suggest_int('n_estimators', 700, 3000),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'device':"gpu",
'gpu_use_dp':True
}
X_train, X_test, y_train, y_test = train_test_split(X, y.flatten() , test_size=0.1)
 lgbm_regr = LGBMRegressor(**param)
 lgbm_regr = lgbm_regr.fit(X_train, y_train, verbose=False)
 score = mean_squared_error(lgbm_regr.predict(X_test), y_test, squared=False)
 return score
missing_data_train = getNulls(final_train)
 missing_data_train.head(434 ).T<drop_column>
study = optuna.create_study(direction='minimize',sampler=TPESampler())
 study.optimize(lambda trial : objectiveLGBM(trial, X, y, X_test), n_trials=20)
 print('Best trial: score {},
params {}'.format(study.best_trial.value,study.best_trial.params))
best_param2 = study.best_trial.params
lgbm = LGBMRegressor(**best_param2, device="gpu",gpu_use_dp=True, objective='regression', metric='root_mean_squared_error', learning_rate= 0.01, boosting_type='gbdt')
del missing_data_train<concatenate>
final_model = xgb.XGBRegressor(n_estimators= 2000, max_depth= 16,tree_method='gpu_hist', predictor='gpu_predictor')
 sgd = SGDRegressor(max_iter=1000)
 hgb = HistGradientBoostingRegressor(max_depth=3, min_samples_leaf=1)
 cat = CatBoostRegressor(task_type="GPU", verbose=False)
 estimators = [
lgbm, cat, sgd, hgb, xgbReg
]
S_train, S_test = stacking(estimators, X, y, X_test, regression=True, metric=mean_squared_error, n_folds=5,
shuffle=False, random_state=0, verbose=2)
 final_model.fit(S_train, y)
 print(mean_squared_error(final_model.predict(S_train), y, squared=False))
ntrain = final_train.shape[0]
ntest = final_test.shape[0]
all_data = pd.concat([final_train, final_test], axis = 0, sort = False )<drop_column>
submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 y_hat = final_model.predict(S_test)
 submission["target"] = y_hat
submission[["target"]].to_csv("/kaggle/working/submission_stacking.csv")
 joblib.dump(final_model, '/kaggle/working/skacking.pkl' )
all_data = all_data.drop(columns=['isFraud'] )<drop_column>
submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 lgbm = LGBMRegressor(**best_param2, device="gpu",gpu_use_dp=True, objective='regression', learning_rate= 0.01, metric='root_mean_squared_error', boosting_type='gbdt')
 lgbm = lgbm.fit(X, y, verbose=False)
 y_hat = lgbm.predict(submission.to_numpy())
 print(mean_squared_error(lgbm.predict(X), y, squared=False))
submission["target"] = y_hat
submission[["target"]].to_csv("/kaggle/working/submission_lgbm.csv")
 joblib.dump(lgbm, '/kaggle/working/lgbm.pkl' )
<define_variables><EOS> 
submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv", index_col=["id"])
 params = {'n_estimators': 3520, 'max_depth': 11, 'min_child_weight': 231, 'gamma': 2, 'colsample_bytree': 0.7, 'lambda': 0.014950936465569798, 'alpha': 0.28520156840812494, 'subsample': 0.6}
xgbReg = train(xgb.XGBRegressor(**params, tree_method='gpu_hist', random_state=42, predictor='gpu_predictor', learning_rate=0.01, nthread=-1))
y_hat = xgbReg.predict(submission.to_numpy())
 print(mean_squared_error(xgbReg.predict(X), y, squared=False))
submission["target"] = y_hat
submission[["target"]].to_csv("/kaggle/working/submission_xgb.csv")
 joblib.dump(xgbReg, '/kaggle/working/xgb_reg.pkl' )