Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.datasets import mnist
from keras.models import Model, Sequential
from keras.models import load_model
from keras.layers import Input, Conv2D, MaxPooling2D, BatchNormalization
from keras.layers import UpSampling2D, Dropout, Dense, Flatten
from keras.callbacks import TensorBoard
if TUNING_XGB:
model = XGBRegressor(
 tree_method='gpu_hist',
predictor='gpu_predictor',
n_jobs=4,
**trial.params)
 else:
params = {
'max_depth': 5,
'n_estimators': 7779,
'eta': 0.0044144556312306175,
'subsample': 0.30000000000000004,
'colsample_bytree': 0.2,
'colsample_bylevel': 0.4,
'min_child_weight': 0.21792841014662054,
'reg_lambda': 5.06808562586094,
'reg_alpha': 0.036826697275635915,
'gamma': 0.002452743312016066,
}
model = XGBRegressor(
 tree_method='gpu_hist',
predictor='gpu_predictor',
n_jobs=4,
**params)
 avg_rmse, oof_preds, preds = cv_loop(
 X_train=X_train,
y_train=y_train,
X_test=X_test,
model=model,
useful_features=all_features,
num_features=num_features,
cat_features=cat_features,
cat_features_to_encode=cat_features_to_encode,
encoding = encoding,
new_features=new_features,
scaling = scaling,
clip = clip, clipmin=np.log(34900), clipmax=np.log(755000),
tuning=False,
early_stopping = early_stopping
)<save_to_csv>
train_dir = ".. /input/train.csv"
test_dir = ".. /input/test.csv"
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = preds
submission.to_csv('xgb_preds.csv', index = False)
 oof_preds.to_csv('xgb_oof_preds.csv', header=False )<categorify>
df_train = pd.read_csv(train_dir)
 df_train.info()
_, _, X_train, y_train, X_test, all_features, num_features, _, _, cat_features, cat_features_to_encode = load_data()
useful_features = [e for e in all_features if e not in('PoolArea','3SsnPorch','MoSold','YrSold','RoofMatl','Utilities','MiscFeatureGar2','PoolQC')]
new_features = ['NbNAs','LivLotRatio','Spaciousness','MedNhbdArea','GrLivAreaInNbhd','MedNhbdArea_Ext','LotAreaInNbhd','OverallQualCondProduct','LowQualFinRatio']
encoding = 'ohe'
scaling = True
clip = True
early_stopping = False<compute_train_metric>
y_train = df_train['label']
X_train = df_train.drop(columns=['label'] )
def objective(trial):
param_grid = {
'alpha': trial.suggest_loguniform('alpha', 0.0001, 10000),
'max_iter': trial.suggest_loguniform('max_iter', 1000, 900000),
'random_state' : 42
}
model = Lasso(
 **param_grid 
)
 avg_rmse, _, _ = cv_loop(
 X_train = X_train,
y_train = y_train,
X_test = X_test,
model = model,
useful_features = useful_features,
num_features = num_features,
cat_features = cat_features,
cat_features_to_encode = cat_features_to_encode,
encoding = encoding,
new_features = new_features,
scaling = scaling,
clip = clip, clipmin=np.log(34900), clipmax=np.log(755000),
tuning = True,
early_stopping = early_stopping 
)
 return avg_rmse<find_best_params>
display_image(X_train, y_train, n=10, label=True )
if TUNING_LASSO:
study = optuna.create_study(direction='minimize', study_name=STUDY_NAME)
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: ', len(study.trials))
print('Best trial:')
 trial = study.best_trial
print('\tValue: {}'.format(trial.value))
print('\tParams: ')
 for key, value in trial.params.items() :
print('\t\t{}: {}'.format(key, value))<train_on_grid>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, shuffle=False )
if TUNING_LASSO:
model = Lasso(**trial.params, random_state=42)
 else:
params = {
'alpha': 0.0018185000964940012,
'max_iter': 21098,
'random_state' : 42
}
model = Lasso(**params)
 avg_rmse, oof_preds, preds = cv_loop(
 X_train = X_train,
y_train = y_train,
X_test = X_test,
model = model,
useful_features = useful_features,
num_features = num_features,
cat_features = cat_features,
cat_features_to_encode = cat_features_to_encode,
encoding = encoding,
new_features = new_features,
scaling = scaling,
clip = clip, clipmin = np.log(34900), clipmax = np.log(755000),
tuning = False,
early_stopping = early_stopping
)<save_to_csv>
X_train = X_train.values.reshape(-1, 28,28,1)
 X_val = X_val.values.reshape(-1, 28,28,1 )
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = preds
submission.to_csv('lasso_preds.csv', index = False)
 oof_preds.to_csv('lasso_oof_preds.csv', header=False )<load_from_csv>
X_train = X_train / 255.0
X_val = X_val / 255.0
X_train, y_train = oof_preds[['xgb_oof_preds','lasso_oof_preds']], oof_preds['y_train']
xgb_preds = pd.read_csv('xgb_preds.csv' ).iloc[:,1]
lasso_preds = pd.read_csv('lasso_preds.csv' ).iloc[:,1]
X_test = pd.concat([np.log(xgb_preds), np.log(lasso_preds)], axis=1 )<find_best_model_class>
Y_train = pd.get_dummies(y_train ).values
Y_val = pd.get_dummies(y_val ).values
metamodel = LinearRegression()
cum_rmse_val = 0
iteration = 1
kf = KFold(n_splits=5, shuffle=True, random_state=42)
 for train_index, val_index in kf.split(X_train, y_train):
X_train_, X_val_ = X_train.iloc[train_index], X_train.iloc[val_index]
y_train_, y_val_ = y_train[train_index], y_train[val_index]
metamodel.fit(X_train_, y_train_)
 y_val_preds = metamodel.predict(X_val_)
 rmse_val = mean_squared_error(y_val_, y_val_preds, squared=False)
 print(rmse_val)
 cum_rmse_val = cum_rmse_val + rmse_val
new_preds=metamodel.predict(X_test)
 if(iteration==1):
preds = new_preds
else:
preds = preds + new_preds
iteration = iteration+1
print(cum_rmse_val/5)
 preds = preds / 5<save_to_csv>
print("La valeur {} est encodée vers le vecteur {}".format(y_train[0], Y_train[0]))
print("valeur {} transformée en vecteur: {}".format(y_train[20], Y_train[20]))
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = np.exp(preds)
 submission.to_csv('stacking_preds.csv', index = False )<import_modules>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=20,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5, 5), activation='relu', padding='Same', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(5, 5), activation='relu', padding='Same'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3, 3), activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(strides=(2,2)))
 model.add(Dropout(0.25))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<count_missing_values>
model.add(Flatten())
 model.add(Dense(units=1024, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(units=1024, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(units=10, activation='softmax'))
Null_train = train.isnull().sum()
Null_train[Null_train > 0]<drop_column>
model.compile(loss='categorical_crossentropy', optimizer = Adam(lr=0.0001), metrics=["accuracy"] )
train.drop(['Alley', 'PoolQC', 'Fence', 'MiscFeature', 'Id'], axis = 1, inplace = True )<define_variables>
hist = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=32),
steps_per_epoch=1000,
epochs=25,
verbose=1,
validation_data=(X_val, Y_val))
Null_train_data = train[['LotFrontage','MasVnrType', 'MasVnrArea', 'FireplaceQu',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Electrical', 'GarageType', 'GarageYrBlt',
'GarageFinish', 'GarageQual', 'GarageCond']]<count_unique_values>
final_loss, final_acc = model.evaluate(X_val, Y_val, verbose=0)
 print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
def view_null_data(data):
return pd.DataFrame({"Data Type":data.dtypes, "Unique Count":data.apply(lambda x: x.nunique() ,axis=0),
"Null Count": data.isnull().sum() } )<feature_engineering>
Y_hat = model.predict(X_val)
 Y_hat[0]
train['LotFrontage'] = train['LotFrontage'].fillna(train.LotFrontage.mean())
 train['GarageYrBlt'] = train['GarageYrBlt'].fillna(train.GarageYrBlt.mean())
 train['MasVnrArea'] = train['MasVnrArea'].fillna(train.MasVnrArea.mode() [0] )<count_missing_values>
Y_pred = np.argmax(Y_hat, axis=1)
 Y_true = np.argmax(Y_val, axis=1 )
Null_test = test.isnull().sum()
Null_test[Null_test > 0]<drop_column>
cm = confusion_matrix(Y_true, Y_pred)
 print(cm )
Id = test['Id']
test.drop(['Alley', 'PoolQC', 'Fence', 'MiscFeature', 'Id'], axis = 1, inplace = True )<drop_column>
X_test = pd.read_csv(test_dir)
 X_test = X_test.values.reshape(-1, 28,28,1)
 X_test = X_test / 255.0
Null_test_data = test[['MSZoning', 'LotFrontage', 'Utilities', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'MasVnrArea', 'BsmtQual','FireplaceQu',
'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1',
'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF',
'BsmtFullBath', 'BsmtHalfBath', 'KitchenQual', 'Functional',
'GarageType', 'GarageYrBlt', 'GarageFinish', 'GarageQual',
'GarageCars','GarageArea','GarageQual', 'GarageCond', 'SaleType']]<feature_engineering>
Y_hat = model.predict(X_test, verbose=1)
 Y_pred = np.argmax(Y_hat, axis=1 )
test['LotFrontage'] = test['LotFrontage'].fillna(test['LotFrontage'].mean())
 test['BsmtFinSF1'] = test['BsmtFinSF1'].fillna(test['BsmtFinSF1'].mean())
 test['BsmtUnfSF'] = test['BsmtUnfSF'].fillna(test['BsmtUnfSF'].mean())
 test['TotalBsmtSF'] = test['TotalBsmtSF'].fillna(test['TotalBsmtSF'].mean())
 test['GarageYrBlt'] = test['GarageYrBlt'].fillna(test['GarageYrBlt'].mean())
 test['GarageArea'] = test['GarageArea'].fillna(test['GarageArea'].mean())
 test['MasVnrArea'] = test['MasVnrArea'].fillna(test['MasVnrArea'].mode() [0])
 test['BsmtFullBath'] = test['BsmtFullBath'].fillna(test['BsmtFullBath'].mode() [0])
 test['BsmtFinSF2'] = test['BsmtFinSF2'].fillna(test['BsmtFinSF2'].mode() [0])
 test['BsmtHalfBath'] = test['BsmtHalfBath'].fillna(test['BsmtHalfBath'].mode() [0])
 test['LotFrontage'] = test['LotFrontage'].fillna(test['LotFrontage'].mode() [0] )<compute_test_metric>
display_image(pd.DataFrame(X_test.reshape(-1, 784)) , Y_pred, n=10, label=True )
high_corr_col = filter_correlation(train, 0.7)
 high_corr_col<drop_column>
results = pd.Series(Y_pred, name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_predictions.csv",index=False )