Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<init_hyperparams>
import datetime
import imageio
import numpy as np
import pandas as pd
import pickle
from sklearn.model_selection import train_test_split
from fastai.vision import *
from fastai.metrics import accuracy, error_rate
from fastai.widgets import DatasetFormatter, PredictionsCorrector
fold_num = 10
EARLY_STOPPING_ROUNDS = 10
VERBOSE_EVAL = 10000
LGB_ROUND_NUM = 10000
objective = 'regression'
metric = 'rmse'
params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': objective,
'metric': metric,
'verbosity': -1,
"seed": 42,
}
@contextmanager
def timer(logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None):
if prefix: format_str = str(prefix)+ format_str
if suffix: format_str = format_str + str(suffix)
 start = time()
yield
d = time() - start
out_str = format_str.format(d)
 if logger:
logger.info(out_str)
 else:
print(out_str)
 def fit_lgbm(X, y, cv, params: dict=None, verbose: int=50):
metric_func = mean_squared_error
if params is None:
params = {}
models = []
oof_pred = np.zeros_like(y, dtype=np.float)
 for i,(idx_train, idx_valid)in enumerate(cv):
x_train, y_train = X[idx_train], y[idx_train]
x_valid, y_valid = X[idx_valid], y[idx_valid]
lgb_train = lgb.Dataset(x_train, y_train)
 lgb_valid = lgb.Dataset(x_valid, y_valid)
 with timer(prefix='fit fold={} '.format(i + 1)) :
lgb_model = lgb_tuner.train(params, lgb_train,
num_boost_round=LGB_ROUND_NUM,
valid_names=["train", "valid"],
valid_sets=[lgb_train, lgb_valid],
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose_eval=VERBOSE_EVAL)
 pred_i = lgb_model.predict(x_valid)
 oof_pred[idx_valid] = pred_i
models.append(lgb_model)
 print(f'Fold {i} RMSE: {metric_func(y_valid, pred_i)**.5:.4f}')
 score = metric_func(y, oof_pred)**.5
print('FINISHED | Whole RMSE: {:.4f}'.format(score))
return oof_pred, models<train_model>
print(fastai.__version__ )
fold = KFold(n_splits=5, shuffle=True, random_state=71)
 cv = list(fold.split(X, y))
oof, models = fit_lgbm(X.values, y, cv, params=params )<create_dataframe>
np.random.seed(42 )
def visualize_importance(models, feat_train_df):

 feature_importance_df = pd.DataFrame()
for i, model in enumerate(models):
_df = pd.DataFrame()
_df['feature_importance'] = model.feature_importance()
_df['column'] = feat_train_df.columns
_df['fold'] = i + 1
feature_importance_df = pd.concat([feature_importance_df, _df], axis=0, ignore_index=True)
 order = feature_importance_df.groupby('column')\ 
.sum() [['feature_importance']]\ 
.sort_values('feature_importance', ascending=False ).index[:50]
fig, ax = plt.subplots(figsize=(max(6, len(order)*.4), 7))
sns.boxenplot(data=feature_importance_df, x='column', y='feature_importance', order=order, ax=ax, palette='viridis')
 ax.tick_params(axis='x', rotation=90)
 ax.grid()
fig.tight_layout()
return fig, ax<split>
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8)
 def opt(trial):
n_estimators = trial.suggest_int('n_estimators', 0, 1000)
 max_depth = trial.suggest_int('max_depth', 1, 20)
 learning_rate = trial.suggest_discrete_uniform('learning_rate', 0.01,0.1,0.01)
 min_child_weight = trial.suggest_int('min_child_weight', 1, 20)
 subsample = trial.suggest_discrete_uniform('subsample', 0.5, 0.9, 0.1)
 colsample_bytree = trial.suggest_discrete_uniform('colsample_bytree', 0.5, 0.9, 0.1)
 xgboost_tuna = xgb.XGBRegressor(
 random_state=42,
n_estimators = n_estimators,
max_depth = max_depth,
min_child_weight = min_child_weight,
subsample = subsample,
colsample_bytree = colsample_bytree, 
)
 xgboost_tuna.fit(X_train,y_train)
 tuna_pred_test = xgboost_tuna.predict(X_test)
 return mean_squared_error(y_test, tuna_pred_test )<train_model>
y = train["label"].values
X = train.iloc[:, 1:].values
X.shape, y.shape
def fit_xgb(X, y, cv, params: dict=None, verbose: int=50):
metric_func = mean_squared_error
if params is None:
params = {}
models = []
oof_pred = np.zeros_like(y, dtype=np.float)
 for i,(idx_train, idx_valid)in enumerate(cv):
x_train, y_train = X[idx_train], y[idx_train]
x_valid, y_valid = X[idx_valid], y[idx_valid]
model_xgb = xgb.XGBRegressor(**params)
 with timer(prefix='fit fold={} '.format(i + 1)) :
model_xgb.fit(x_train, y_train, eval_set=[(x_valid, y_valid)],verbose=-1)
 pred_i = model_xgb.predict(x_valid)
 oof_pred[idx_valid] = pred_i
models.append(model_xgb)
 print(f'Fold {i} RMSE: {metric_func(y_valid, pred_i)**.5:.4f}')
 score = metric_func(y, oof_pred)**.5
print('FINISHED | Whole RMSE: {:.4f}'.format(score))
return oof_pred, models<define_search_space>
X_train, X_valid, y_train, y_valid = \
train_test_split(X, y, test_size=0.05, random_state=42, stratify=y )
params_xgb = {'n_estimators': 208,
'max_depth': 4,
'learning_rate':0.08,
'min_child_weight': 13,
'subsample': 0.8,
'colsample_bytree': 0.8}
oof_xgb, models_xgb = fit_xgb(X.values, y, cv, params=params_xgb )<set_options>
def to_img_shape(X, y=[]):
"Format matrix of rows, Nx784 to Nx28x28x3"
X = np.array(X ).reshape(-1,28,28)
 X = np.stack(( X,)*3, axis=-1)
 y = np.array(y)
 return X, y
def save_imgs(path:Path, data, labels=[]):

 path.mkdir(parents=True, exist_ok=True)
 for label in np.unique(labels): 
(path / str(label)).mkdir(parents=True, exist_ok=True)
 for i in range(len(data)) :
if(len(labels)!= 0):
imageio.imsave(str(path / str(labels[i])/(str(i)+ '.jpg')) , data[i])
 else:
imageio.imsave(str(path / f'testimg_{str(i ).zfill(5)}.jpg'), data[i] )
def opt_cb(trial):
params = {
'iterations' : trial.suggest_int('iterations', 50, 300),
'depth' : trial.suggest_int('depth', 4, 10),
'learning_rate' : trial.suggest_loguniform('learning_rate', 0.01, 0.3),
'random_strength' :trial.suggest_int('random_strength', 0, 100),
'bagging_temperature' :trial.suggest_loguniform('bagging_temperature', 0.01, 100.00),
'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']),
'od_wait' :trial.suggest_int('od_wait', 10, 50)
 }
train_pool = Pool(X_train, y_train)
 test_pool = Pool(X_test, y_test)
 catboost_tuna = CatBoostRegressor(**params)
 catboost_tuna.fit(train_pool)
 tuna_pred_test = catboost_tuna.predict(test_pool)
 pred_labels = np.rint(tuna_pred_test)
 return mean_squared_error(y_test, pred_labels )<train_model>
X_train, y_train = to_img_shape(X_train, y_train)
 X_valid, y_valid = to_img_shape(X_valid, y_valid)
 X_test, _ = to_img_shape(test )
def fit_cb(X, y, cv, params: dict=None, verbose: int=50):
metric_func = mean_squared_error
if params is None:
params = {}
models = []
oof_pred = np.zeros_like(y, dtype=np.float)
 for i,(idx_train, idx_valid)in enumerate(cv):
x_train, y_train = X[idx_train], y[idx_train]
x_valid, y_valid = X[idx_valid], y[idx_valid]
train_pool = Pool(x_train, label = y_train)
 valid_pool = Pool(x_valid, label = y_valid)
 model_cb = CatBoost(params)
 with timer(prefix='fit fold={} '.format(i + 1)) :
model_cb.fit(train_pool,
eval_set = valid_pool,
use_best_model = True,
silent = True,
plot = False)
 print(model_cb.get_best_score())
 pred_i = model_cb.predict(x_valid)
 oof_pred[idx_valid] = pred_i
models.append(model_cb)
 print(f'Fold {i} RMSE: {metric_func(y_valid, pred_i)**.5:.4f}')
 score = metric_func(y, oof_pred)**.5
print('FINISHED | Whole RMSE: {:.4f}'.format(score))
return oof_pred, models<train_model>
%%time
save_imgs(Path('/kaggle/working/data/train'), X_train, y_train )
params_cb = {
'loss_function': 'RMSE',
'max_depth': 3,
'learning_rate': 0.08,
'subsample': 0.8,
'num_boost_round': 1000,
'early_stopping_rounds': 100,
}
oof_cb, models_cb = fit_cb(X.values, y, cv, params=params_cb )<drop_column>
%%time
save_imgs(Path('/kaggle/working/data/valid'), X_valid, y_valid )
df_test = df_test.drop("id",axis=1 )<predict_on_test>
%%time
save_imgs(Path('/kaggle/working/data/test'), X_test )
pred_lgb = np.array([model.predict(df_test.values)for model in models])
 pred_lgb = np.mean(pred_lgb, axis=0)
 pred_lgb = np.where(pred_lgb < 0, 0, pred_lgb)
 pred_xgb = np.array([model.predict(df_test.values)for model in models_xgb])
 pred_xgb = np.mean(pred_xgb, axis=0)
 pred_xgb = np.where(pred_xgb < 0, 0, pred_xgb)
 pred_cb = np.array([model.predict(df_test.values)for model in models_cb])
 pred_cb = np.mean(pred_cb, axis=0)
 pred_cb = np.where(pred_cb < 0, 0, pred_cb)
 tmp_sub = pd.DataFrame({"lgb":pred_lgb,
"xgb":pred_xgb,
"cb":pred_cb})
 tmp_sub["pred"] = tmp_sub.mean(axis="columns" )<prepare_output>
path = Path('/kaggle/working/data')
 image_list =(ImageList.from_folder(path)
.split_by_folder() 
.label_from_folder())
 data =(image_list.databunch(bs=1)
.normalize(imagenet_stats))
submission["target"] = tmp_sub["pred"].copy()<save_to_csv>
tfms = get_transforms()
tfms[0]
submission.to_csv("submission.csv", index=False )<save_to_csv>
tfms[1]
submission.to_csv("submission.csv", index=False )<create_dataframe>
tfms = get_transforms(do_flip=False)
 tfms[0]
oof_df = pd.DataFrame({"lgb":oof,
"xgb":oof_xgb,
"cb":oof_cb})
 oof_df["pred"] = oof_df.mean(axis="columns" )<import_modules>
image_list =(image_list.transform(get_transforms(do_flip=False), size=28)
.add_test(ItemList.from_folder(path=path/"test"), label=None))
data =(image_list.databunch(bs=256)
.normalize(imagenet_stats))
import numpy as np
import pandas as pd<load_from_csv>
learn = cnn_learner(data, models.resnet18, metrics=accuracy )
train_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 print('Train: ', train_data.shape)
 print('Test: ', test_data.shape )<prepare_x_and_y>
learn.model
y = train_data['target']
X = train_data.drop(columns=['target', 'id'])
 X_test = test_data.drop(columns='id' )<split>
learn.lr_find()
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.05, random_state=22 )<init_hyperparams>
learn.fit_one_cycle(5, 0.02 )
params = {'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'boosting_type': 'gbdt',
'feature_pre_filter': False,
'learning_rate': 0.007,
'num_leaves': 102,
'min_child_samples': 20,
'sub_feature': 0.4,
'sub_row': 1,
'subsample_freq': 0,
'lambda_l1': 4.6,
'lambda_l2': 1.9}
N_FOLDS = 10
kf = KFold(n_splits = N_FOLDS)
 oof = np.zeros(len(y))
oof_vanilla = np.zeros(len(y))
preds = np.zeros(len(X_test))
params['learning_rate'] = 0.005
params['num_iterations'] = 5000
for train_ind, test_ind in tqdm(kf.split(X)) :
Xtrain = X.iloc[train_ind]
Xval = X.iloc[test_ind]
ytrain = y.iloc[train_ind]
yval = y.iloc[test_ind]
model = LGBMRegressor(**params)
 vanilla_model = LGBMRegressor()
model.fit(Xtrain, ytrain, eval_set =(( Xval,yval)) , early_stopping_rounds = 50, verbose = 0)
 vanilla_model.fit(Xtrain, ytrain)
 p = model.predict(Xval)
 p_vanilla = vanilla_model.predict(Xval)
 oof[test_ind] = p
oof_vanilla[test_ind] = p_vanilla
preds += model.predict(X_test)/N_FOLDS
print(f'mean square error on training data(vanilla model): {np.round(mean_squared_error(y, oof_vanilla, squared=False),5)}')
 print(f'mean square error on training data(with tuning): {np.round(mean_squared_error(y, oof, squared=False),5)}' )<save_to_csv>
learn.save('stage-1' )
output = pd.DataFrame({"id":test_data.id, "target":preds})
 output.to_csv('submission.csv', index=False )<train_model>
learn.load('stage-1');
print('Finish!' )<set_options>
learn.unfreeze()
learn.lr_find()
warnings.filterwarnings('ignore')
 RANDOM_SEED = 123<load_from_csv>
%%time
learn.fit_one_cycle(35, max_lr=slice(1e-5, 0.02/10))
train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 sample = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv" )<feature_engineering>
learn.save('stage-2' )
train['magic1'] = train['cont10']/train['cont11']
train['magic2'] = train['cont11']/train['cont10']
train['magic3'] = train['cont1']/train['cont7']
train['magic4'] = train['cont7']/train['cont1']
train['magic5'] = train['cont4']/train['cont6']
test['magic1'] = test['cont10']/test['cont11']
test['magic2'] = test['cont11']/test['cont10']
test['magic3'] = test['cont1']/test['cont7']
test['magic4'] = test['cont7']/test['cont1']
test['magic5'] = test['cont4']/test['cont6']<prepare_x_and_y>
interp = ClassificationInterpretation.from_learner(learn )
train = train.drop('id', axis=1)
 test = test.drop('id', axis=1)
 X = train.drop('target', axis=1)
 y = train.target<choose_model_class>
def make_submission_file(
 learner,
filename=f'submission_{datetime.datetime.now().strftime("%Y%m%d_%H%M%S")}.csv',
preds=None 
):

 if preds is None:
preds, _ = learner.get_preds(ds_type=DatasetType.Test)
 preds = np.argmax(preds, 1)
 test_index = []
num = len(learn.data.test_ds)
 for i in range(num):
test_index.append(str(learner.data.test_ds.items[i] ).split('/')[-1])
 df =(pd.DataFrame(data={"Label": preds, "Filename": test_index})
.sort_values(by='Filename')
.drop('Filename', axis=1)
.assign(ImageId = range(1, len(preds)+ 1)) 
.reset_index(drop=True)
.reindex(columns=['ImageId', 'Label']))
df.to_csv(filename, index=False)
 print(f'Saved predictions as {filename}.' )
cat = CatBoostRegressor(iterations=1000 )<compute_train_metric>
make_submission_file(learn, filename="resnet18-fine-tuned.csv" )
model = [cat]
for mod in model:
score = cross_val_score(mod, X, y, cv=3, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("CAT RMSE Mean Score: ", np.mean(score))<compute_train_metric>
most_unsure = DatasetFormatter.from_most_unsure(learn )
model = [cat]
for mod in model:
score = cross_val_score(mod, X, y, cv=10, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("CAT RMSE Mean Score: ", np.mean(score))<choose_model_class>
err1 = 1 - 0.99442
err2 = 1 - 0.99571
print(f'Human in the loop improvement: {100*(err1-err2)/err1}%' )