Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<import_modules>
%matplotlib inline
print(pd.__version__, np.__version__, keras.__version__ )
<init_hyperparams>
data = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 data.describe()
<train_model>
X, y = data.iloc[:, 1:] / 255, data.iloc[:, 0]
X, y = X.values, y.values
print(X.dtype, y.dtype)
 X.resize(X.shape[0], 28, 28, 1)
 y = to_categorical(y, num_classes=10)
 print(type(X), X.shape)
 print(type(y), y.shape )
%%time
df_train, df_test = preprocess_data()
X_train, X_test = create_features(df_train, df_test)
 y_train = df_train.loc[:, "SalePrice"]
xgb_params = {'max_depth': 4, 'learning_rate': 0.00875, 'n_estimators': 3515, 'min_child_weight': 2, 'colsample_bytree': 0.2050378195385253, 'subsample': 0.40369887914955715, 'reg_alpha': 0.3301567121037565, 'reg_lambda': 0.046181862052743}
xgb = XGBRegressor(**xgb_params, random_state=42)
 print("
X_train shape: ", X_train.shape)
 print("X_test shape: ", X_test.shape)
 print("score: ", score_dataset(X_train, y_train, xgb))<save_to_csv>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=3)
 print(X_train.shape, y_train.shape)
 print(X_val.shape, y_val.shape )
df_train, df_test = preprocess_data()
X_train, X_test = create_features(df_train, df_test)
 y_train = df_train.loc[:, "SalePrice"]
xgb = XGBRegressor(**xgb_params, random_state=42)
 xgb.fit(X_train, np.log(y_train))
predictions = np.exp(xgb.predict(X_test))
output = pd.DataFrame({'Id': X_test.index, 'SalePrice': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=0.1 
)
 datagen.fit(X_train )
pd.set_option('display.max_columns', 500)
 pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn<load_from_csv>
regularizer=10E-4
def res_block(x, i, regularizer=regularizer):
x1 = keras.layers.Conv2D(pow(2, i),(3, 3), activation='relu',
padding='same', kernel_initializer="he_normal",
kernel_regularizer=keras.regularizers.l2(regularizer))(x)
 x2 = keras.layers.Conv2D(pow(2, i),(3, 3), activation='relu',
padding='same', kernel_initializer="he_normal",
kernel_regularizer=keras.regularizers.l2(regularizer))(x1)
 x2 = keras.layers.Conv2D(pow(2, i),(3, 3), activation='relu',
padding='same', kernel_initializer="he_normal",
kernel_regularizer=keras.regularizers.l2(regularizer))(x2)
 x = keras.layers.concatenate([x1, x2])
 return x
input_X = keras.layers.Input(shape=(28, 28, 1))
x = input_X
for i in range(3, 7):
x = res_block(x, i)
 x = keras.layers.MaxPooling2D(2, 2 )(x)
 x = keras.layers.BatchNormalization()(x)
 x = keras.layers.Flatten()(x)
 y = keras.layers.Dense(units=10, activation='softmax' )(x)
 model = keras.models.Model(inputs=input_X, outputs=y)
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
 model.summary()
plot_model(model, show_shapes=True)
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 train.head(5 )<prepare_x_and_y>
es = keras.callbacks.EarlyStopping(monitor='val_loss', patience=7, restore_best_weights=True)
 history = model.fit_generator(
 datagen.flow(X_train, y_train, batch_size=64),
epochs=32,
verbose=1,
validation_data=(X_val, y_val),
shuffle=True,
callbacks=[es]
)
train_id = train["Id"]
test_id = test["Id"]
train.drop("Id", axis=1, inplace=True)
 test.drop("Id", axis=1, inplace=True)
 train.drop(train[(train['GrLivArea'] > 4000)&(train['SalePrice'] < 300000)].index, axis=0, inplace=True)
 train['SalePrice'] = np.log1p(train['SalePrice'])
 ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train['SalePrice'].values
all_data = pd.concat([train, test] ).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None")
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0)
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data.drop(['Utilities'], axis=1, inplace=True)
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
 all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for col in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[col].values))
all_data[col] = lbl.transform(list(all_data[col].values))
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness = skewness[abs(skewness)> 0.75]
skewed_features = skewness.index
lam = 0.15
for feature in skewed_features:
all_data[feature] = boxcox1p(all_data[feature], lam)
 all_data = pd.get_dummies(all_data)
 train = all_data[:ntrain]
test = all_data[ntrain:]<compute_train_metric>
test_data = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 test_imgs = test_data / 255
test_imgs = test_imgs.values
test_imgs.resize(test_imgs.shape[0], 28, 28, 1)
 print(test_imgs.shape )
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=1 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, random_state =7,
verbosity=0, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 models = [('Lasso', lasso),('ElasticNet', ENet),('Kernel Ridge', KRR), 
('Gradient Boosting', GBoost),('Xgboost', model_xgb),('LGBM', model_lgb)]
for model in models:
score = rmsle_cv(model[1])
 print('{} score: {}({})'.format(model[0], score.mean() , score.std()))<choose_model_class>
predictions = model.predict(test_imgs ).argmax(axis=1)
 print(predictions.shape)
 print(predictions )
class AveragingModel(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, x, y):
self.models_ = [clone(model)for model in self.models]
for model in self.models_:
model.fit(x, y)
 return self
def predict(self, x):
predictions = np.column_stack([
model.predict(x)for model in self.models_
])
 return np.mean(predictions, axis=1)
 averaging_model = AveragingModel(models=[ENet, GBoost, KRR, lasso])
 score = rmsle_cv(averaging_model)
 print("Averaging model score: {:.4f}({:.4f})".format(score.mean() , score.std()))
class StackingModel(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, x, y):
self.base_models_ = [list() for model in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=1)
 out_of_fold_predictions = np.zeros(( x.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(x, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(x[train_index], y[train_index])
 y_pred = instance.predict(x[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, x):
meta_features = np.column_stack([
np.column_stack([model.predict(x)for model in trained_base_models] ).mean(axis=1)
 for trained_base_models in self.base_models_])
 return self.meta_model_.predict(meta_features)
 stacking_model = StackingModel(base_models=[ENet, GBoost, KRR], meta_model=lasso)
 score = rmsle_cv(stacking_model)
 print("Stacking model score: {:.4f}({:.4f})".format(score.mean() , score.std()))<save_to_csv>
results = pd.DataFrame({"ImageID": range(1, len(test_imgs)+ 1),
"Label": predictions})
 print(results)
 results.to_csv("sample_submission.csv", header=True, index=False )