Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.models import Sequential
from keras.layers import Conv2D, Lambda, MaxPooling2D
from keras.layers import Dense, Dropout, Flatten
from keras.layers.normalization import BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.utils.np_utils import to_categorical
def feature_engineering(data):
new_data = data.copy()
new_data['cat2p6'] = new_data['cat2'] + new_data['cat6']
new_data['cat6p1'] = new_data['cat6'] + new_data['cat1']
new_data['cat2p1'] = new_data['cat2'] + new_data['cat1']
new_data['cont0p8'] = new_data['cont0']*new_data['cont8']
new_data['cont0p5'] = new_data['cont0']*new_data['cont5']
new_data['cont11p8'] = new_data['cont11']*new_data['cont8']
return new_data
new_train = feature_engineering(train)
 new_test = feature_engineering(test)
 print(new_train.shape)
 print(new_test.shape )<compute_train_metric>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 print("Data are Ready!!" )
print("lgb base CV scores label")
 model = lgb.LGBMRegressor(**lgb_params_base)
 lgb_train_oof, lgb_test_preds, lgb_all_scores = kfold_CV_pipe(train, target, test, model,enc_method = 'label', columns = train.columns)
 oof_score = rmse(target, lgb_train_oof)
 print(f"lgb oof score: {oof_score:.6f}")
 print("---------------------------------------------------------------")
 print("lgb base CV scores glmm")
 model = lgb.LGBMRegressor(**lgb_params_base)
 lgb_train_oof, lgb_test_preds, lgb_all_scores = kfold_CV_pipe(train, target, test, model,enc_method = 'glmm' ,columns = train.columns)
 oof_score = rmse(target, lgb_train_oof)
 print(f"lgb oof score: {oof_score:.6f}")
 print("---------------------------------------------------------------")
 print("lgb base new features CV scores")
 model = lgb.LGBMRegressor(**lgb_params_base)
 lgb_train_oof, lgb_test_preds, lgb_all_scores = kfold_CV_pipe(new_train, target, new_test, model,enc_method = 'glmm', columns = new_train.columns)
 oof_score = rmse(target, lgb_train_oof)
 print(f"lgb oof score: {oof_score:.6f}" )<compute_train_metric>
print(f"Training data size is {train.shape}
Testing data size is {test.shape}" )
print("---------------------------------------------------------------")
 print("===============================================================")
 print("---------------------------------------------------------------")
 print("lgb1 CV scores")
 model = lgb.LGBMRegressor(**lgb_params)
 lgb1_train_oof, lgb1_test_preds, lgb1_all_scores = kfold_CV_pipe(train, target, test, model,enc_method = 'glmm', columns = train.columns)
 oof_score = rmse(target, lgb1_train_oof)
 print(f"lgb1 oof score: {oof_score:.6f}")
 print("---------------------------------------------------------------")
 print("===============================================================")
 print("---------------------------------------------------------------")
 print("lgb2 CV scores")
 model = lgb.LGBMRegressor(**lgb_params_2)
 lgb2_train_oof, lgb2_test_preds, lgb2_all_scores = kfold_CV_pipe(train, target, test, model,enc_method = 'glmm', columns = train.columns)
 oof_score = rmse(target, lgb2_train_oof)
 print(f"lgb2 oof score: {oof_score:.6f}")
 print("---------------------------------------------------------------")
 print("===============================================================")
 print("---------------------------------------------------------------")
 print("xgb CV scores")
 model = xgb.XGBRegressor(**xgb_params)
 xgb_train_oof, xgb_test_preds, xgb_all_scores = kfold_CV_pipe(train, target, test, model,enc_method = 'glmm', columns = train.columns)
 oof_score = rmse(target, xgb_train_oof)
 print(f"xgb oof score: {oof_score:.6f}" )<save_to_csv>
X = train.drop(['label'], 1 ).values
y = train['label'].values
test_x = test.values
sub = smpl_sub.copy()
sub['target'] = lgb1_test_preds
sub.to_csv("lgb1_submission.csv", index=False)
<save_to_csv>
X = X / 255.0
test_x = test_x / 255.0
sub = smpl_sub.copy()
sub['target'] = lgb2_test_preds
sub.to_csv("lgb2_submission.csv", index=False )<save_to_csv>
y = to_categorical(y)
 print(f"Label size {y.shape}" )
sub = smpl_sub.copy()
sub['target'] = xgb_test_preds
sub.to_csv("xgb_submission.csv", index=False )<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0 )
weights = [0.2, 0.4, 0.4]
oof_pred_wavg = weights[0]*lgb1_train_oof + weights[1]*lgb2_train_oof + weights[2]*xgb_train_oof
oof_score_wavg = rmse(target, oof_pred_wavg)
 print(f"oof score weighted avg: {oof_score_wavg:.6f}")
 test_pred_wavg = weights[0]*lgb1_test_preds + weights[1]*lgb2_test_preds + weights[2]*xgb_test_preds
sub = smpl_sub.copy()
sub['target'] = test_pred_wavg
sub.to_csv("wavg_submission.csv", index=False )<import_modules>
mean = np.mean(X_train)
 std = np.std(X_train)
 def standardize(x):
return(x-mean)/std
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import optuna
from lightgbm import LGBMRegressor
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split,KFold
from sklearn.preprocessing import LabelEncoder<load_from_csv>
epochs = 50
batch_size = 64
train=pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test=pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv' )<categorify>
model=Sequential()
model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu", input_shape=(28,28,1)))
 model.add(Conv2D(filters=64, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(Conv2D(filters=128, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters=256, kernel_size =(3,3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(512,activation="relu"))
model.add(Dense(10,activation="softmax"))
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"] )
cat_var=[f'cat{i}' for i in range(10)]
cont_var=[f'cont{i}' for i in range(14)]
columns=[ col for col in train.columns.tolist() if col not in ['id','target']]
for cat in cat_var:
le = LabelEncoder()
train[cat]=le.fit_transform(train[cat])
 test[cat]=le.transform(test[cat] )<prepare_x_and_y>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 train_gen = datagen.flow(X_train, y_train, batch_size=batch_size)
 test_gen = datagen.flow(X_test, y_test, batch_size=batch_size )
X=train[columns]
y=train.target<init_hyperparams>
history = model.fit_generator(train_gen,
epochs = epochs,
steps_per_epoch = X_train.shape[0] // batch_size,
validation_data = test_gen,
validation_steps = X_test.shape[0] // batch_size )
lgb_params={'random_state': 2021,
'metric': 'rmse',
'n_estimators': 30000,
'n_jobs': -1,
'cat_feature': [x for x in range(len(cat_var)) ],
'bagging_seed': 2021,
'feature_fraction_seed': 2021,
'learning_rate': 0.003899156646724397,
'max_depth': 99,
'num_leaves': 63,
'reg_alpha': 9.562925363678952,
'reg_lambda': 9.355810045480153,
'colsample_bytree': 0.2256038826485174,
'min_child_samples': 290,
'subsample_freq': 1,
'subsample': 0.8805303688019942,
'max_bin': 882,
'min_data_per_group': 127,
'cat_smooth': 96,
'cat_l2': 19
}<init_hyperparams>
y_pred = model.predict(X_test)
 X_test__ = X_test.reshape(X_test.shape[0], 28, 28)
 fig, axis = plt.subplots(4, 4, figsize=(12, 14))
for i, ax in enumerate(axis.flat):
ax.imshow(X_test__[i], cmap='binary')
 ax.set(title = f"Real Number is {y_test[i].argmax() }
Predict Number is {y_pred[i].argmax() }");
f1= 0.6547870667136243
f2= 2.6711351556035487
f3= 20
f4= 49
f5= 2<train_model>
pred = model.predict_classes(test_x, verbose=1 )
%%time
kf=KFold(n_splits=5,random_state=48,shuffle=True)
 preds = np.zeros(test.shape[0])
 rmse=[]
i=0
for idx_train,idx_test in kf.split(X,y):
X_train,X_test=X.iloc[idx_train],X.iloc[idx_test]
y_train,y_test=y.iloc[idx_train],y.iloc[idx_test]
model=LGBMRegressor(**lgb_params)
 model.fit(X_train,y_train,eval_set=(X_test,y_test),early_stopping_rounds=300,verbose=False,eval_metric='rmse')
 predictions=model.predict(X_test,num_iteration=model.best_iteration_)
 rmse.append(mean_squared_error(y_test,predictions,squared=False))
print('First Round:')
 print(f'RMSE {rmse[i]}')
 rmse_tuned=[]
params = lgb_params.copy()
for t in range(1,17):
if t >2:
params['reg_lambda'] *= f1
params['reg_alpha'] += f2
params['num_leaves'] += f3
params['min_child_samples'] -= f4
params['cat_smooth'] -= f5
params['learning_rate']=0.003
if params['min_child_samples']<1:
params['min_child_samples']=1
if t>11:
params['learning_rate']=0.001
model=LGBMRegressor(**params ).fit(X_train,y_train,eval_set=(X_test,y_test),eval_metric='rmse',early_stopping_rounds=200,verbose=False,init_model=model)
 predictions=model.predict(X_test, num_iteration= model.best_iteration_)
 rmse_tuned.append(mean_squared_error(y_test,predictions,squared=False))
print(f'RMSE tuned {t}: {rmse_tuned[t-1]}')
 print(f'Improvement of {rmse[i]-rmse_tuned[t-1]}')
 preds+=model.predict(test[columns],num_iteration=model.best_iteration_)/kf.n_splits
i+=1<save_to_csv>
sub['Label'] = pred
sub.to_csv("CNN_keras_sub.csv", index=False)
 sub.head()