Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<prepare_x_and_y>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from skimage import color
from skimage import measure
from skimage.filters import try_all_threshold
from skimage.filters import threshold_otsu
from skimage.filters import threshold_local
import keras
from keras import Sequential
from keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization
from keras.optimizers import Adam
from keras.utils import to_categorical
from keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint
n_fold = 10
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = train.columns.values
oof = np.zeros(len(train))
LGBMpredictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train, target.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = train.iloc[trn_idx], train.iloc[val_idx]
y_tr, y_val = target.iloc[trn_idx], target.iloc[val_idx]
model = LGBMRegressor(**params, n_estimators = 20000)
 model.fit(X_tr, y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=400)
 oof[val_idx] = model.predict(X_val, num_iteration=model.best_iteration_)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 LGBMpredictions += model.predict(test, num_iteration=model.best_iteration_)/ folds.n_splits 
<choose_model_class>
df_train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 df_test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
<init_hyperparams>
y_train = df_train['label']
X_train = df_train.drop('label', axis = 1)
 X_test = np.array(df_test )
XGparams={'colsample_bytree': 0.7,
'learning_rate': 0.01,
'max_depth': 7,
'min_child_weight': 1,
'n_estimators': 4000,
'nthread': 4,
'objective': 'reg:squarederror',
'subsample': 0.7}<train_model>
y_train = to_categorical(y_train, num_classes = 10)
 y_train.shape
n_fold = 10
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42)
 train_columns = train.columns.values
oof = np.zeros(len(train))
XGpredictions = np.zeros(len(test))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train, target.values)) :
strLog = "fold {}".format(fold_)
 print(strLog)
 X_tr, X_val = train.iloc[trn_idx], train.iloc[val_idx]
y_tr, y_val = target.iloc[trn_idx], target.iloc[val_idx]
model = XGBRegressor(**XGparams)
 model.fit(X_tr, y_tr,
eval_set=[(X_tr, y_tr),(X_val, y_val)], verbose=1000, early_stopping_rounds=400)
 oof[val_idx] = model.predict(X_val, ntree_limit=model.best_iteration)
 preds = model.predict(test, ntree_limit=model.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = train_columns
fold_importance_df["importance"] = model.feature_importances_[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 XGpredictions += model.predict(test, ntree_limit=model.best_iteration)/ folds.n_splits 
<load_from_csv>
X_train, X_val, y_train, y_val = train_test_split(X_train,
y_train,
test_size=0.25,
random_state=1 )
submission = pd.read_csv(input_path / 'sample_submission.csv', index_col='id')
 submission.reset_index(inplace=True)
 submission = submission.rename(columns = {'index':'id'} )<save_to_csv>
kernel_ =(5,5 )
LGBMsubmission=submission.copy()
LGBMsubmission['target'] = LGBMpredictions
LGBMsubmission.to_csv('submission_LGBM.csv', header=True, index=False)
 LGBMsubmission.head()<save_to_csv>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28, 28, 1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dense(256, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.summary()
XGBoostsubmission=submission.copy()
XGBoostsubmission['target'] = XGpredictions
XGBoostsubmission.to_csv('submission_XGBoost.csv', header=True, index=False)
 XGBoostsubmission.head()<save_to_csv>
aug = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1)
 gen_train = aug.flow(X_train, y_train, batch_size=64)
 gen_val = aug.flow(X_val, y_val, batch_size=64 )
EnsembledSubmission=submission.copy()
EnsembledSubmission['target'] =(LGBMpredictions*0.72 + XGpredictions*0.28)
 EnsembledSubmission.to_csv('ensembled_submission.csv', header=True, index=False)
 EnsembledSubmission.head()<import_modules>
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'] )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import optuna
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_squared_error
from sklearn.base import TransformerMixin
import xgboost as xgb
import lightgbm as lgb<load_from_csv>
checkpoint = tf.keras.callbacks.ModelCheckpoint("weights.hdf5",
monitor='val_accuracy',
verbose=1,
save_best_only=True)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=4,
min_lr=0.00005,
verbose=1)
 early_stop = tf.keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True )
df = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 df.head()<compute_train_metric>
training = model.fit(gen_train, epochs=100, batch_size=512,
validation_data=gen_val,
callbacks = [checkpoint, reduce_lr], verbose = 1 )
def objective_xgb(trial, data, target):
parameters = {
'tree_method': 'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 10.0),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4, 0.5, 0.6, 0.7, 0.8, 1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008, 0.009, 0.01, 0.012, 0.014, 0.016, 0.018, 0.02]),
'n_estimators': 1000,
'max_depth': trial.suggest_categorical('max_depth', [5, 7, 9, 11, 13, 15, 17, 20]),
'random_state': trial.suggest_categorical('random_state', [24, 48, 2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 300),
}
folds = KFold(n_splits=5, random_state=1337, shuffle=True)
 rmse = []
for train_idx, test_idx in folds.split(data, target):
X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
model = xgb.XGBRegressor(**parameters)
 model.fit(X_train, y_train, eval_set=[(X_test, y_test)], early_stopping_rounds=100, verbose=False)
 rmse.append(mean_squared_error(y_test, model.predict(X_test), squared=False))
print(f'Mean RMSE for all the folds: {np.mean(rmse)}')
 return np.mean(rmse )<init_hyperparams>
model.load_weights("weights.hdf5" )
xgb_parameters = {
'objective': 'reg:squarederror',
'tree_method': 'gpu_hist',
'n_estimators': 1000,
'lambda': 7.610705234008646,
'alpha': 0.0019377246932580476,
'colsample_bytree': 0.5,
'subsample': 0.7,
'learning_rate': 0.012,
'max_depth': 20,
'random_state': 24,
'min_child_weight': 229
}<split>
y_test = model.predict(X_test)
 y_pred = np.argmax(y_test, axis=1 )
<init_hyperparams><EOS> 
output_csv = {"ImageId":[*range(1,1+len(y_pred)) ], "Label":y_pred}
Y_pre = pd.DataFrame(output_csv)
 Y_pre.set_index("ImageId", drop=True, append=False, inplace=True)
 Y_pre.to_csv("/kaggle/working/submission.csv" )