Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<define_variables>
warnings.filterwarnings("ignore")
 %matplotlib inline
path = Path('/kaggle/input/planet-understanding-the-amazon-from-space')
 path.ls()<load_from_csv>
features = [feature for feature in train_df.columns if feature not in ['id', 'target']]
X_train = train_df[features]
y_train = train_df['target']
X_test = test_df[features]
df = pd.read_csv(path/'train_v2.csv')
 df.head()<feature_engineering>
%%time
forest_reg = RandomForestRegressor(random_state=121, n_jobs=-1)
 scores = cross_val_score(forest_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=4)
 forest_rmse_scores = np.sqrt(-scores)
 print('Random Forest performance:', forest_rmse_scores)
 print('Random Forest performance_mean:', forest_rmse_scores.mean())
tfms = get_transforms(flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.)<load_from_csv>
%%time
xgb_reg = XGBRegressor(random_state=121, objective = 'reg:squarederror', n_jobs=-1)
 scores = cross_val_score(xgb_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=5, n_jobs=-1)
 xgb_rmse_scores = np.sqrt(-scores)
 print('XGBoost performance:', xgb_rmse_scores)
 print('XGBoost performance_mean:', xgb_rmse_scores.mean())
np.random.seed(42)
 src =(ImageList.from_csv(path, 'train_v2.csv', folder='train-jpg', suffix='.jpg')
.split_by_rand_pct(0.2)
.label_from_df(label_delim=' '))<normalization>
%%time
lgbm_reg = LGBMRegressor(random_state=121)
 scores = cross_val_score(lgbm_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=5, n_jobs=-1)
 lgbm_rmse_scores = np.sqrt(-scores)
 print('LGBM performance:', lgbm_rmse_scores)
 print('LGBM performance_mean:', lgbm_rmse_scores.mean())
data =(src.transform(tfms, size=128)
.databunch().normalize(imagenet_stats))<define_variables>
lasso_reg = Lasso()
scores = cross_val_score(lasso_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=5)
 lasso_rmse_scores = np.sqrt(-scores)
 print('LASSO performance:', lasso_rmse_scores)
 print('LASSO performance_mean:', lasso_rmse_scores.mean() )
data.show_batch(rows=3, figsize=(12,9))<choose_model_class>
ridge_reg = Ridge()
scores = cross_val_score(ridge_reg, X_train, y_train, scoring='neg_mean_squared_error', cv=5)
 ridge_rmse_scores = np.sqrt(-scores)
 print('Ridge performance:', ridge_rmse_scores)
 print('Ridge performance_mean:', ridge_rmse_scores.mean() )
arch = models.resnet50<find_best_params>
def rmse_mean(model):

 rmse_scores_mean = np.sqrt(
 -cross_val_score(
 model, X_train, y_train,
scoring="neg_mean_squared_error",
cv=5, 
)).mean()
return(rmse_scores_mean)
 alphas = [1, 10**-1, 10**-2, 10**-3, 10**-4, 10**-5]
lasso_regs = [rmse_mean(Lasso(alpha = alpha)) for alpha in alphas]
lasso_regs = pd.Series(lasso_regs, index=alphas)
 print("LASSO RMSE loss:")
 print(lasso_regs, "
")
 print("LASSO RMSE best_alpha :", lasso_regs.idxmin())
 print("LASSO RMSE best_score value :", lasso_regs.min() , "
")
 plt.figure(figsize=(10, 5))
plt.plot(lasso_regs)
 plt.grid()
plt.title("LASSO: Validation_score - by regularization strength")
 plt.xlabel("Alpha")
 plt.ylabel("RMSE")
 plt.show()
acc_02 = partial(accuracy_thresh, thresh=0.2)
 f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(data, arch, metrics=[acc_02, f_score],path='.. /working/' )<find_best_params>
X = train_df[features]
y = train_df['target']
learn.lr_find()<init_hyperparams>
def objective(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda':
trial.suggest_loguniform('lambda', 1e-3, 1),
'alpha':
trial.suggest_loguniform('alpha', 1e-3, 1),
'colsample_bytree':
trial.suggest_categorical('colsample_bytree',
[0.1, 0.2, 0.3,0.5,0.7,0.9]),
'subsample':
trial.suggest_categorical('subsample',
[0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate':
trial.suggest_categorical('learning_rate',
[0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'max_depth':
trial.suggest_categorical('max_depth',
[5,7,9,11,13,15,17,20,23,25]),
'random_state': 48,
'min_child_weight': trial.suggest_int('min_child_weight', 1, 400),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],
early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
lr = 0.01<train_model>
study = optuna.create_study(direction='minimize' )
learn.fit_one_cycle(5, slice(lr))<save_model>
study.optimize(objective, n_trials=25 )
learn.save('stage-1-rn50' )<train_model>
print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
learn.fit_one_cycle(5, slice(1e-5, lr/5))<save_model>
study_data_table = study.trials_dataframe()
study_data_table.to_csv('study_xgboost.csv', index=False)
 study_data_table
learn.save('stage-2-rn50' )<categorify>
best_trial_paras = {'tree_method':'gpu_hist', 'lambda': 0.03349655513592068,
'alpha': 0.12097952030992898, 'colsample_bytree': 0.5,
'subsample': 0.4, 'learning_rate': 0.01,
'n_estimators': 4000, 'max_depth': 11, 'min_child_weight': 179,
'random_state': 2021
}
data =(src.transform(tfms, size=256)
.databunch().normalize(imagenet_stats))
learn.data = data
data.train_ds[0][0].shape<find_best_params>
preds = np.zeros(test_df.shape[0])
 kf = KFold(n_splits=5,random_state=48,shuffle=True)
 rmse=[]
n=0
for trn_idx, test_idx in kf.split(train_df[features],train_df['target']):
X_tr,X_val=train_df[features].iloc[trn_idx],train_df[features].iloc[test_idx]
y_tr,y_val=train_df['target'].iloc[trn_idx],train_df['target'].iloc[test_idx]
model = xgb.XGBRegressor(**best_trial_paras)
 model.fit(X_tr,y_tr,eval_set=[(X_val,y_val)],early_stopping_rounds=100, verbose=False)
 preds+=model.predict(test_df[features])/kf.n_splits
rmse.append(mean_squared_error(y_val, model.predict(X_val), squared=False))
print(n+1,rmse[n])
 n+=1
lr=1e-2/2<train_model>
sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv')
 print(sub.head() )
<save_model><EOS> 
sub['target']=preds
print(sub.head())
 sub.to_csv('xgboost_submission.csv', index=False )