Kaggle competition: Digit Recognizer
features_nom = [
"MSSubClass",
"MSZoning",
"Street",
"Alley",
"LandContour",
"LotConfig",
"Neighborhood",
"Condition1",
"Condition2",
"BldgType",
"HouseStyle",
"RoofStyle",
"RoofMatl",
"Exterior1st",
"Exterior2nd",
"MasVnrType",
"Foundation",
"Heating",
"CentralAir",
"GarageType",
"MiscFeature",
"SaleType",
"SaleCondition"
]
five_levels = ["Po", "Fa", "TA", "Gd", "Ex"]
ten_levels = list(range(10))
ordered_levels = {
"OverallQual": ten_levels,
"OverallCond": ten_levels,
"ExterQual": five_levels,
"ExterCond": five_levels,
"BsmtQual": five_levels,
"BsmtCond": five_levels,
"HeatingQC": five_levels,
"KitchenQual": five_levels,
"FireplaceQu": five_levels,
"GarageQual": five_levels,
"GarageCond": five_levels,
"PoolQC": five_levels,
"LotShape": ["Reg", "IR1", "IR2", "IR3"],
"LandSlope": ["Sev", "Mod", "Gtl"],
"BsmtExposure": ["No", "Mn", "Av", "Gd"],
"BsmtFinType1": ["Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"],
"BsmtFinType2": ["Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"],
"Functional": ["Sal", "Sev", "Maj1", "Maj2", "Mod", "Min2", "Min1", "Typ"],
"GarageFinish": ["Unf", "RFn", "Fin"],
"PavedDrive": ["N", "P", "Y"],
"Utilities": ["NoSeWa", "NoSewr", "AllPub"],
"CentralAir": ["N", "Y"],
"Electrical": ["Mix", "FuseP", "FuseF", "FuseA", "SBrkr"],
"Fence": ["MnWw", "GdWo", "MnPrv", "GdPrv"]
}
ordered_levels = {key: ["None"] + value for key, value in
ordered_levels.items() }
def encode(df):
for name in features_nom:
df[name] = df[name].astype("category")
 if "None" not in df[name].cat.categories:
df[name].cat.add_categories("None", inplace=True)
 for name, levels in ordered_levels.items() :
df[name] = df[name].astype(CategoricalDtype(levels,
ordered=True))
return df<categorify>
X_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 X_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print('Shape of the training data: ', X_train.shape)
 print('Shape of the test data: ', X_test.shape )
def impute(df):
for name in df.select_dtypes("number"):
df[name] = df[name].fillna(0)
 for name in df.select_dtypes("category"):
df[name] = df[name].fillna("None")
 return df<split>
y_train = X_train['label']
X_train.drop(labels = ['label'], axis=1, inplace=True )
df_train, df_test = load_data()<compute_train_metric>
print('Null values in training data: ',X_train.isna().any().sum())
 print('Null values in test data: ',X_test.isna().any().sum())
 X_train = X_train / 255.0
X_test = X_test / 255.0
def score_dataset(X, y, model=XGBRegressor()):

 for colname in X.select_dtypes(["category"]):
X[colname] = X[colname].cat.codes
log_y = np.log(y)
 score = cross_val_score(
 model,
X,
log_y,
cv=5,
scoring="neg_mean_squared_error" 
)
 score = -1 * score.mean()
score = np.sqrt(score)
 return score<compute_test_metric>
y_train = to_categorical(y_train, num_classes=10 )
X = df_train.copy()
y = X.pop("SalePrice")
 baseline_score = score_dataset(X, y)
 print(f"Baseline score: {baseline_score:.5f} RMSLE" )<prepare_x_and_y>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1 )
X = df_train.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 mi_scores.head()<drop_column>
model = keras.models.Sequential([layers.Conv2D(32,(3,3), activation="relu",padding='same', input_shape=(28,28,1)) ,
layers.MaxPooling2D(2,2),
layers.Dropout(0.25),
layers.Conv2D(64,(3,3), activation="relu",padding='same'),
layers.MaxPooling2D(2,2),
layers.Dropout(0.25),
layers.Flatten() ,
layers.BatchNormalization() ,
layers.Dense(128, activation="elu", kernel_initializer="he_normal"),
layers.BatchNormalization() ,
layers.Dense(32, activation="elu", kernel_initializer="he_normal"),
layers.BatchNormalization() ,
layers.Dense(10, activation="softmax")])
 model.summary()
def drop_uninformative(df, mi_scores):
return df.loc[:, mi_scores > 0.0]<create_dataframe>
optimizer = Adam(learning_rate=0.001, epsilon=1e-07)
 model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics=['accuracy'])
 earlyStopping = EarlyStopping(monitor='val_accuracy', patience=10, verbose=0, mode='auto')
 mcp = ModelCheckpoint('.mdl_wts.hdf5', save_best_only=True, monitor='val_accuracy', mode='auto')
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='auto' )
X = df_train.copy()
y = X.pop("SalePrice")
 X = drop_uninformative(X, mi_scores)
 score_dataset(X, y )<categorify>
datagen = ImageDataGenerator(
 rotation_range=5,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=5,
zoom_range=0.1)
 datagen.fit(X_train )
def label_encode(df):
X = df.copy()
for colname in X.select_dtypes(["category"]):
X[colname] = X[colname].cat.codes
return X<categorify>
Batch_size=100
Epochs = 100
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=Batch_size),
epochs = Epochs,
validation_data =(X_val,y_val),
verbose = 2,
steps_per_epoch=X_train.shape[0]//Batch_size,
callbacks = [earlyStopping, mcp, reduce_lr_loss] )
def mathematical_transforms(df):
X = pd.DataFrame()
X["LivLotRatio"] = df.GrLivArea / df.LotArea
X["Spaciousness"] =(df.FirstFlrSF + df.SecondFlrSF)/ df.TotRmsAbvGrd
return X
def interactions(df):
X = pd.get_dummies(df.BldgType, prefix="Bldg")
 X = X.mul(df.GrLivArea, axis=0)
 return X
def counts(df):
X = pd.DataFrame()
X["PorchTypes"] = df[[
"WoodDeckSF",
"OpenPorchSF",
"EnclosedPorch",
"Threeseasonporch",
"ScreenPorch"
]].gt(0.0 ).sum(axis=1)
 return X
def break_down(df):
X = pd.DataFrame()
X["MSClass"] = df.MSSubClass.str.split("_", n=1, expand=True)[0]
return X
def group_transforms(df):
X = pd.DataFrame()
X["MedNhbdArea"] = df.groupby("Neighborhood")["GrLivArea"].transform("median")
 X["MedCondQual"] = df.groupby("OverallQual")["OverallCond"].transform("median")
 X["MedBsmt"] = df.groupby("BsmtQual")["TotalBsmtSF"].transform("median")
 return X<create_dataframe>
model.load_weights(filepath = '.mdl_wts.hdf5')
 scores = model.evaluate(X_val, y_val, callbacks = [earlyStopping, mcp, reduce_lr_loss] )
def apply_pca(X, standardize=True):
if standardize:
X =(X - X.mean(axis=0)) / X.std(axis=0)
 pca = PCA()
X_pca = pca.fit_transform(X)
 component_names = [f"PC{i+1}" for i in range(X_pca.shape[1])]
X_pca = pd.DataFrame(X_pca, column=component_names)
 loadings = pd.DataFrame(
 pca.components_.T,
columns=component_names,
index=X.columns, 
)
 return pca, X_pca, loadings
def plot_variance(pca, width=8, dpi=100):
fig, axs = plt.subplots(1, 2)
 n = pca.n_components_
grid = np.arange(1, n + 1)
 evr = pca.explained_variance_ratio_
axs[0].bar(grid, evr)
 axs[0].set(
 xlabel="Component", title="% Explained Variance", ylim=(0.0, 1.0)
)
 cv = np.cumsum(evr)
 axs[1].plot(np.r_[0, grid], np.r_[0, cv], "o-")
 axs[1].set(
 xlabel="Component", title="% Cumulative Variance", ylim=(0.0, 1.0)
)
 fig.set(figwidth=8, dpi=100)
 return axs<feature_engineering>
for layer in model.layers:
if'conv' in layer.name:
filters, biases = layer.get_weights()
print('Layer: ', layer.name, filters.shape)
 print('Filter size:(', filters.shape[0], ',', filters.shape[1], ')')
 print('Channels in this layer: ', filters.shape[2])
 print('Number of filters: ', filters.shape[3])
 count = 1
plt.figure(figsize =(18, 4))
for i in range(filters.shape[3]):
ax= plt.subplot(4, filters.shape[3]/4, count)
 ax.set_xticks([])
 ax.set_yticks([])
 plt.imshow(filters[:,:,0, i], cmap=plt.cm.binary)
 count+=1
plt.show()
def pca_inspired(df):
X = pd.DataFrame()
X["Feature1"] = df.GrLivArea + df.TotalBsmtSF
X["Feature2"] = df.YearRemodAdd * df.TotalBsmtSF
return X
def pca_components(df, features):
X = df.loc[:, features]
_, X_pca, _ = apply_pca(X)
 return X_pca
pca_features = [
"GarageArea",
"YearRemodAdd",
"TotalBsmtSF",
"GrLivArea",
]<create_dataframe>
print('total number of layers',len(model.layers))
def indicate_outliers(df):
X_new = pd.DataFrame()
X_new["Outlier"] =(df.Neighborhood == "Edwards")&(df.SaleCondition == "Partial")
 return X_new<categorify>
layer_outputs = [layer.output for layer in model.layers[0:6]]
activation_model = models.Model(inputs = model.input, outputs = layer_outputs )
class CrossFoldEncoder:
def __init__(self, encoder, **kwargs):
self.encoder_ = encoder
self.kwargs_ = kwargs
self.cv_ = KFold(n_splits=5)
 def fit_transform(self, X, y, cols):
self.fitted_encoders_ = []
self.cols_ = cols
X_encoded = []
for idx_encode, idx_train in self.cv_.split(X):
fitted_encoder = self.encoder_(cols=cols, **self.kwargs_)
 fitted_encoder.fit(
 X.iloc[idx_encode, :], y.iloc[idx_encode], 
)
 X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])
 self.fitted_encoders_.append(fitted_encoder)
 X_encoded = pd.concat(X_encoded)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded
def transform(self, X):
X_encoded_list = []
for fitted_encoder in self.fitted_encoders_:
X_encoded = fitted_encoder.transform(X)
 X_encoded_list.append(X_encoded[self.cols_])
 X_encoded = reduce(
 lambda x, y: x.add(y, fill_value=0), X_encoded_list 
)/ len(X_encoded_list)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded<categorify>
img_tensor = X_test[4].reshape(-1, 28, 28, 1)
 activations = activation_model.predict(img_tensor )
<prepare_x_and_y><EOS> 
submissions = pd.DataFrame({"ImageId": list(range(1,len(test_pred)+1)) ,
"Label": test_pred})
 submissions.to_csv("submission.csv", index=False, header=True )