Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_from_csv>
config = tf.ConfigProto(device_count = {'GPU': 1 , 'CPU': 4})
 sess = tf.Session(config=config)
 keras.backend.set_session(sess)
 K.set_image_dim_ordering('tf' )
def load_data() :
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 X_train = train.copy()
y_train = X_train['SalePrice']
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 X_test = test.copy()
X_train = clean(X_train)
 X_test = clean(X_test)
 all_features = ['MSSubClass','MSZoning','LotFrontage','LotArea','Street','Alley','LotShape','LandContour','Utilities','LotConfig','LandSlope','Neighborhood','Condition1','Condition2','BldgType','HouseStyle','OverallQual','OverallCond','YearBuilt','YearRemodAdd','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','MasVnrArea','ExterQual','ExterCond','Foundation','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','Heating','HeatingQC','CentralAir','Electrical','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','KitchenQual','TotRmsAbvGrd','Functional','Fireplaces','FireplaceQu','GarageType','GarageYrBlt','GarageFinish','GarageCars','GarageArea','GarageQual','GarageCond','PavedDrive','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','PoolQC','Fence','MoSold','YrSold','SaleType','SaleCondition','SalePrice','MiscFeatureShed','MiscFeatureGar2','MiscFeatureOthr']
num_features = ['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageCars','LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscFeatureGar2','MiscFeatureOthr','MiscFeatureShed','MoSold','YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']
num_continuous_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea', 'GarageArea','WoodDeckSF','OpenPorchSF','EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscFeatureGar2','MiscFeatureOthr','MiscFeatureShed','MoSold','YearBuilt','YearRemodAdd','YrSold','GarageYrBlt']
num_discrete_features=['BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageCars']
cat_features = ['LotShape','LandContour','OverallQual','OverallCond','Utilities','LandSlope','ExterQual','ExterCond','BsmtQual','BsmtCond','BsmtExposure','HeatingQC','KitchenQual','FireplaceQu','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MSSubClass','MSZoning','Street','Alley','LotConfig','Neighborhood','Condition1','Condition2','BldgType','HouseStyle','RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','Foundation','BsmtFinType1','BsmtFinType2','Heating','CentralAir','Electrical','Functional','GarageType','PavedDrive','SaleType','SaleCondition']
cat_features_to_encode = cat_features.copy()
cat_features_to_encode.remove('OverallQual')
 cat_features_to_encode.remove('OverallCond')
 return train, test, X_train, y_train, X_test, all_features, num_features, num_continuous_features, num_discrete_features, cat_features, cat_features_to_encode<train_on_grid>
train_data = np.genfromtxt('.. /input/train.csv', delimiter=',')[1:]
train_X = train_data[:, 1:]
train_y_orig = train_data[:, :1]
train_y = np.zeros([train_y_orig.shape[0], 10])
 for ind in range(train_y_orig.shape[0]):
train_y[ind][int(train_y_orig[ind][0])] = 1
def cv_loop(
 X_train,
y_train,
X_test,
model,
useful_features,
num_features,
cat_features,
cat_features_to_encode,
encoding = 'ohe',
new_features=[],
scaling=False,
clip=False, clipmin=np.log(34900), clipmax=np.log(755000),
tuning=True,
early_stopping=True 
):
y_train = np.log(y_train)
 num_features = np.intersect1d(num_features, useful_features)
 cat_features = np.intersect1d(cat_features, useful_features)
 cat_features_to_encode = np.intersect1d(cat_features_to_encode, useful_features)
 cum_rmse_val = 0
iteration = 1
N_SPLITS = 5
kf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=42)
 for train_index, val_index in kf.split(X_train, y_train):
X_train_, X_val_ = X_train.iloc[train_index], X_train.iloc[val_index]
y_train_, y_val_ = y_train[train_index], y_train[val_index]
X_train__ = X_train_.reset_index(drop=True)
 X_val__ = X_val_.reset_index(drop=True)
 X_train__ = X_train__.loc[:, useful_features]
X_val__ = X_val__.loc[:, useful_features]
if(tuning==False):
useful_features_ = useful_features.copy()
useful_features_.remove('SalePrice')
 X_test__ = X_test.loc[:, useful_features_]
if(new_features.count('NbNAs')==1):
X_train__['NbNAs'] = X_train__.isnull().sum(axis=1)
 X_val__['NbNAs'] = X_val__.isnull().sum(axis=1)
 if(tuning==False): X_test__['NbNAs'] = X_test__.isnull().sum(axis=1)
 for col in num_features:
X_train__[col] = X_train__[col].fillna(0)
 X_val__[col] = X_val__[col].fillna(0)
 if(tuning==False): X_test__[col] = X_test__[col].fillna(0)
 for col in cat_features:
X_train__[col] = X_train__[col].fillna("None")
 X_val__[col] = X_val__[col].fillna("None")
 if(tuning==False): X_test__[col] = X_test__[col].fillna("None")
 if(new_features.count('LivLotRatio')==1):
X_train__['LivLotRatio'] = X_train__['GrLivArea'] / X_train__['LotArea']
X_val__['LivLotRatio'] = X_val__['GrLivArea'] / X_val__['LotArea']
if(tuning==False): X_test__['LivLotRatio'] = X_test__['GrLivArea'] / X_test__['LotArea']
if(new_features.count('Spaciousness')==1):
X_train__['Spaciousness'] = X_train__['GrLivArea'] / X_train__['TotRmsAbvGrd']
X_val__['Spaciousness'] = X_val__['GrLivArea'] / X_val__['TotRmsAbvGrd']
if(tuning==False): X_test__['Spaciousness'] = X_test__['GrLivArea'] / X_test__['TotRmsAbvGrd']
if(new_features.count('MedNhbdArea')==1):
feat = X_train__.groupby('Neighborhood')['GrLivArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'MedNhbdArea'] = X_train__['Neighborhood'].map(feat)
 X_val__.loc[:,'MedNhbdArea'] = X_val__['Neighborhood'].map(feat)
 if(tuning==False): X_test__.loc[:,'MedNhbdArea'] = X_test__['Neighborhood'].map(feat)
 if(new_features.count('GrLivAreaInNbhd')==1):
feat = X_train__.groupby('Neighborhood')['GrLivArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'GrLivAreaInNbhd'] = X_train__['Neighborhood'].map(feat)
 X_train__['GrLivAreaInNbhd'] = X_train__['GrLivArea'] - X_train__['GrLivAreaInNbhd']
X_val__.loc[:,'GrLivAreaInNbhd'] = X_val__['Neighborhood'].map(feat)
 X_val__['GrLivAreaInNbhd'] = X_val__['GrLivArea'] - X_val__['GrLivAreaInNbhd']
if(tuning==False):
X_test__.loc[:,'GrLivAreaInNbhd'] = X_test__['Neighborhood'].map(feat)
 X_test__['GrLivAreaInNbhd'] = X_test__['GrLivArea'] - X_test__['GrLivAreaInNbhd']
if(new_features.count('MedNhbdArea_Ext')==1):
feat = X_train__.groupby('Neighborhood')['LotArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'MedNhbdArea_Ext'] = X_train__['Neighborhood'].map(feat)
 X_val__.loc[:,'MedNhbdArea_Ext'] = X_val__['Neighborhood'].map(feat)
 if(tuning==False): X_test__.loc[:,'MedNhbdArea_Ext'] = X_test__['Neighborhood'].map(feat)
 if(new_features.count('LotAreaInNbhd')==1):
feat = X_train__.groupby('Neighborhood')['LotArea'].median()
feat = feat.to_dict()
X_train__.loc[:,'LotAreaInNbhd'] = X_train__['Neighborhood'].map(feat)
 X_train__['LotAreaInNbhd'] = X_train__['LotArea'] - X_train__['LotAreaInNbhd']
X_val__.loc[:,'LotAreaInNbhd'] = X_val__['Neighborhood'].map(feat)
 X_val__['LotAreaInNbhd'] = X_val__['LotArea'] - X_val__['LotAreaInNbhd']
if(tuning==False):
X_test__.loc[:,'LotAreaInNbhd'] = X_test__['Neighborhood'].map(feat)
 X_test__['LotAreaInNbhd'] = X_test__['LotArea'] - X_test__['LotAreaInNbhd']
if(new_features.count('OverallQualCondProduct')==1):
X_train__['OverallQualCondProduct'] = X_train__['OverallQual'] * X_train__['OverallCond']
X_val__['OverallQualCondProduct'] = X_val__['OverallQual'] * X_val__['OverallCond']
if(tuning==False): X_test__['OverallQualCondProduct'] = X_test__['OverallQual'] * X_test__['OverallCond']
if(new_features.count('LowQualFinRatio')==1):
X_train__['LowQualFinRatio'] = X_train__['LowQualFinSF'] / X_train__['GrLivArea']
X_val__['LowQualFinRatio'] = X_val__['LowQualFinSF'] / X_val__['GrLivArea']
if(tuning==False): X_test__['LowQualFinRatio'] = X_test__['LowQualFinSF'] / X_test__['GrLivArea']
if(encoding=='ohe'):
enc = OneHotEncoder(handle_unknown = 'ignore')
 X_train__enc = pd.DataFrame(enc.fit_transform(X_train__[cat_features_to_encode] ).toarray())
 X_val__enc = pd.DataFrame(enc.transform(X_val__[cat_features_to_encode] ).toarray())
 X_train__enc.columns = enc.get_feature_names(cat_features_to_encode)
 X_val__enc.columns = enc.get_feature_names(cat_features_to_encode)
 X_train__ = X_train__.join(X_train__enc)
 X_val__ = X_val__.join(X_val__enc)
 if(tuning==False):
X_test__enc = pd.DataFrame(enc.transform(X_test__[cat_features_to_encode] ).toarray())
 X_test__enc.columns = enc.get_feature_names(cat_features_to_encode)
 X_test__ = X_test__.join(X_test__enc)
 elif(encoding=='ord'):
enc = OrdinalEncoder()
X_train__enc = pd.DataFrame(enc.fit_transform(X_train__[cat_features_to_encode]))
X_val__enc = pd.DataFrame(enc.transform(X_val__[cat_features_to_encode]))
X_train__enc.columns = [cat_features_to_encode]
X_val__enc.columns = [cat_features_to_encode]
X_train__ = X_train__.join(X_train__enc, rsuffix='_ord_enc')
 X_val__ = X_val__.join(X_val__enc, rsuffix='_ord_enc')
 if(tuning==False):
X_test__enc = pd.DataFrame(enc.transform(X_test__[cat_features_to_encode]))
X_test__enc.columns = [cat_features_to_encode]
X_test__ = X_test__.join(X_test__enc, rsuffix='_ord_enc')
 elif(encoding=='tar_enc'):
for f in cat_features_to_encode:
feat = X_train__.groupby(f)['SalePrice'].mean()
feat = feat.to_dict()
X_train__.loc[:,f"tar_enc_{f}"] = X_train__[f].map(feat)
 X_val__.loc[:,f"tar_enc_{f}"] = X_val__[f].map(feat)
 if(tuning==False): X_test__.loc[:,f"tar_enc_{f}"] = X_test__[f].map(feat)
 X_train__.drop(columns='SalePrice', inplace=True)
 X_train__.drop(columns=cat_features_to_encode, inplace=True)
 X_val__.drop(columns='SalePrice', inplace=True)
 X_val__.drop(columns=cat_features_to_encode, inplace=True)
 if(tuning==False): X_test__.drop(columns=cat_features_to_encode, inplace=True)
 X_train__.to_csv(f'X_train__{iteration}.csv', index = False)
 X_val__.to_csv(f'X_val__{iteration}.csv', index = False)
 if(tuning==False): X_test__.to_csv(f'X_test__{iteration}.csv', index = False)
 if scaling:
scaler = StandardScaler()
X_train__ = pd.DataFrame(scaler.fit_transform(X_train__), columns=X_train__.columns)
 X_val__ = pd.DataFrame(scaler.transform(X_val__), columns=X_val__.columns)
 if(tuning==False): X_test__ = pd.DataFrame(scaler.transform(X_test__), columns=X_test__.columns)
 X_train__.to_csv(f'X_train__{iteration}__scaled.csv', index = False)
 X_val__.to_csv(f'X_val__{iteration}__scaled.csv', index = False)
 if(tuning==False): X_test__.to_csv(f'X_test__{iteration}__scaled.csv', index = False)
 if early_stopping:
model.fit(X_train__, y_train_, eval_set=[(X_val__, y_val_)], early_stopping_rounds=100, verbose=False)
 else:
model.fit(X_train__, y_train_)
 y_val_preds = model.predict(X_val__)
 if(clip): y_val_preds = np.clip(y_val_preds,clipmin,clipmax)
 if(tuning==False):
if iteration==1:
oof_preds = pd.Series(data=y_val_preds,index=val_index)
 else:
oof_preds = pd.concat([oof_preds, pd.Series(data=y_val_preds,index=val_index)])
 rmse_val = mean_squared_error(y_val_, y_val_preds, squared=False)
 print(str(iteration)+ '/' + str(N_SPLITS)+ ' KFold RMSLE: ' + str(rmse_val))
cum_rmse_val = cum_rmse_val + rmse_val
if(tuning==False):
new_preds = model.predict(X_test__)
 if(clip): new_preds = np.clip(new_preds,clipmin,clipmax)
 if iteration==1:
preds = new_preds
else:
preds = preds + new_preds
iteration = iteration + 1
if(tuning==False): preds = preds/N_SPLITS
avg_rmse = cum_rmse_val/N_SPLITS
print('Average RMSLE: ' + str(avg_rmse))
if tuning:
return avg_rmse, None, None
else:
return avg_rmse, np.exp(oof_preds.sort_index()), np.exp(preds )<split>
model = Sequential()
input_shape =(28, 28, 1)
 model.add(Conv2D(32, kernel_size=(7, 7), padding='same', activation='relu', input_shape=input_shape))
model.add(Conv2D(32, kernel_size=(7, 7), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size=(5, 5), strides=(2,2), padding='same', activation='relu'))
model.add(Conv2D(64, kernel_size=(5, 5), strides=(2,2), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size=(3, 3), strides=(3,3), padding='same', activation='relu'))
model.add(Conv2D(128, kernel_size=(3, 3), strides=(3,3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.6))
model.add(Dense(10, activation='softmax'))
opt = RMSprop()
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'] )
_, _, X_train, y_train, X_test, all_features, num_features, num_continuous_features, num_discrete_features, cat_features, cat_features_to_encode = load_data()
X_train.head(2 )<create_dataframe>
batch_size=2**8
epochs = 100
n = train_X.shape[0]
train_X = train_X.reshape(( n, 28, 28, 1)).astype('float32')/ 255
history = model.fit(train_X, train_y,
epochs=epochs,
batch_size=batch_size )
feature_types = pd.DataFrame(data=[num_features, cat_features])
 feature_types.index=['Numerical','Categorical']
feature_types.style.set_table_styles([
{'selector': 'thead', 'props': [('display', 'none')]}
] )<concatenate>
print(model.evaluate(train_X, train_y))
f_with_na_train = X_train.isna().sum(axis=0)
 f_with_na_train = f_with_na_train[f_with_na_train>0]
f_with_na_train.name='Nb of NaNs in train'
f_with_na_test = X_test.isna().sum(axis=0)
 f_with_na_test = f_with_na_test[f_with_na_test>0]
f_with_na_test.name='Nb of NaNs in test'
f_with_na = pd.concat([f_with_na_train, f_with_na_test], axis=1)
 f_with_na.fillna(0, inplace=True)
 f_with_na = f_with_na[['Nb of NaNs in train','Nb of NaNs in test']]
f_with_na.sort_values(['Nb of NaNs in train', 'Nb of NaNs in test'], ascending=False)[['Nb of NaNs in train', 'Nb of NaNs in test']].plot(kind='bar', figsize=(20,4))
plt.show()<define_variables>
test_data = np.genfromtxt('.. /input/test.csv', delimiter=',')[1:]
_, _, X_train, y_train, X_test, all_features, num_features, _, _, cat_features, cat_features_to_encode = load_data()
useful_features = [e for e in all_features if e not in('PoolArea','3SsnPorch','MoSold','YrSold','RoofMatl','Utilities','MiscFeatureGar2','PoolQC')]
new_features = ['NbNAs','LivLotRatio','Spaciousness','MedNhbdArea','GrLivAreaInNbhd','MedNhbdArea_Ext','LotAreaInNbhd','OverallQualCondProduct','LowQualFinRatio']
encoding = 'tar_enc'
scaling = True
clip = False
early_stopping = True<define_search_space>
predictions = model.predict(test_data.reshape(( test_data.shape[0], 28, 28, 1)).astype('float32')/ 255)
 predictions = predictions.argmax(1 )
<find_best_params><EOS> 
sub_data = np.zeros([predictions.shape[0], 2])
 count = 0
for val in predictions:
sub_data[count] = [count + 1, val]
count += 1
sub_data = sub_data.astype(int)
 np.savetxt(fname="submission.csv",
X=sub_data,
fmt='%i',
delimiter=',',
comments='',
header='ImageId,Label' )