Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<sort_values>
import PIL
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import keras
from matplotlib import pyplot
from sklearn import preprocessing
component = "PC1"
idx = X_pca[component].sort_values(ascending=False ).index
df_train[["SalePrice", "Neighborhood", "SaleCondition"] + pca_features].iloc[idx]<create_dataframe>
run_model1 = False
run_model2 = False
run_model3 = False
run_model_adv = True
def indicate_outliers(df):
X_new = pd.DataFrame()
X_new["Outlier"] =(df.Neighborhood == "Edwards")&(df.SaleCondition == "Partial")
 return X_new<categorify>
train = pd.read_csv('.. /input/train.csv', delimiter=',')
 test = pd.read_csv('.. /input/test.csv', delimiter=',' )
class CrossFoldEncoder:
def __init__(self, encoder, **kwargs):
self.encoder_ = encoder
self.kwargs_ = kwargs
self.cv_ = KFold(n_splits=5)
 def fit_transform(self, X, y, cols):
self.fitted_encoders_ = []
self.cols_ = cols
X_encoded = []
for idx_encode, idx_train in self.cv_.split(X):
fitted_encoder = self.encoder_(cols=cols, **self.kwargs_)
 fitted_encoder.fit(
 X.iloc[idx_encode, :], y.iloc[idx_encode], 
)
 X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])
 self.fitted_encoders_.append(fitted_encoder)
 X_encoded = pd.concat(X_encoded)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded
def transform(self, X):
X_encoded_list = []
for fitted_encoder in self.fitted_encoders_:
X_encoded = fitted_encoder.transform(X)
 X_encoded_list.append(X_encoded[self.cols_])
 X_encoded = reduce(
 lambda x, y: x.add(y, fill_value=0), X_encoded_list 
)/ len(X_encoded_list)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded<drop_column>
train_size = train.shape[0]
test_size = test.shape[0]
X_train = train.iloc[:, 1:].values.astype('uint8')
 Y_train = train.iloc[:, 0]
X_test = test.iloc[:, :].values.astype('uint8')
 img_dimension = np.int32(np.sqrt(X_train.shape[1]))
img_rows, img_cols = img_dimension, img_dimension
nb_of_color_channels = 1
if(keras.backend.image_dim_ordering() =="th"):
X_train = X_train.reshape(train.shape[0], nb_of_color_channels, img_rows, img_cols)
 X_test = X_test.reshape(test.shape[0], nb_of_color_channels, img_rows, img_cols)
 in_shape =(nb_of_color_channels, img_rows, img_cols)
 else:
X_train = X_train.reshape(train.shape[0], img_rows, img_cols, nb_of_color_channels)
 X_test = X_test.reshape(test.shape[0], img_rows, img_cols, nb_of_color_channels)
 in_shape =(img_rows, img_cols, nb_of_color_channels)
 print('Data Information
')
 print('Training set size: {}
Testing set size: {}'.format(train_size, test_size))
print('Image dimension: {0}*{0}'.format(img_dimension))
def create_features(df, df_test=None):
X = df.copy()
y = X.pop('SalePrice')
 mi_scores = make_mi_scores(X, y)
 if df_test is not None:
X_test = df_test.copy()
y_test = X_test.pop("SalePrice")
 X = pd.concat([X, X_test])
 X = X.join(mathematical_transforms(X))
X = X.join(counts(X))
X = X.join(group_transforms(X))
X = X.join(pca_inspired(X))
X = label_encode(X)
 if df_test is not None:
mi_scores = make_mi_scores(X, pd.concat([y, y_test]))
else:
mi_scores = make_mi_scores(X, y)
 X = drop_uninformative(X, mi_scores, 0.02)
 if df_test is not None:
X_test = X.loc[df_test.index, :]
X.drop(df_test.index, inplace=True)
 encoder = CrossFoldEncoder(MEstimateEncoder, m=1)
 X = X.join(encoder.fit_transform(X, y, cols=["MSSubClass"]))
if df_test is not None:
X_test = X_test.join(encoder.transform(X_test))
if df_test is not None:
return X, X_test
else:
return X<prepare_x_and_y>
X_train = X_train.astype('float32')
 X_test = X_test.astype('float32')
 X_train_nor = X_train / 255
X_test_nor= X_test / 255
df_train, df_test = load_data()
X_train = create_features(df_train)
 y_train = df_train.loc[:, 'SalePrice']
score_dataset(X_train, y_train )<prepare_x_and_y>
oh_encoder = preprocessing.OneHotEncoder(categories='auto')
 oh_encoder.fit(Y_train.values.reshape(-1,1))
Y_train_oh = oh_encoder.transform(Y_train.values.reshape(-1,1)).toarray()
X_train = create_features(df_train)
 y_train = df_train.loc[:, "SalePrice"]
xgb_params = dict(
 max_depth=4,
learning_rate=0.0058603076512435655,
n_estimators=5045,
min_child_weight=2,
colsample_bytree=0.22556099175248345,
subsample=0.5632348136091383,
reg_alpha=0.09888625622197889,
reg_lambda=0.00890758697724437,
num_parallel_tree=1, 
)
 xgb = XGBRegressor(**xgb_params)
 score_dataset(X_train, y_train, xgb )<init_hyperparams>
print('One-hot:')
 print(Y_train_oh[:5])
 print('
Label:')
 print(Y_train[:5] )
<predict_on_test>
to_categorical(Y_train, Y_train.unique().shape[0])[:5]
X_train, X_test = create_features(df_train, df_test)
 y_train = df_train.loc[:, "SalePrice"]
xgb = XGBRegressor(**xgb_params)
 xgb.fit(X_train, np.log(y))
predictions = np.exp(xgb.predict(X_test))
output = pd.DataFrame({'Id': X_test.index, 'SalePrice': predictions} )<save_to_csv>
from keras.layers import Activation,Dropout,Dense,Conv2D,AveragePooling2D,Flatten,ZeroPadding2D,MaxPooling2D
from keras.models import Sequential
from keras import optimizers
from keras.callbacks import ReduceLROnPlateau
output.to_csv('submission.csv', index=False)
 print("Your predictions are successfully saved!" )<save_to_csv>
def build_lenet5(model, input_shape=X_train.shape[1:], dropout=0):
S = [1,2,1,2,1]
N_input = [28,28,14,10,5]
P = [2,0,0,0,0]
N = [28,14,10,5,1]
F = [i[0] + 2*i[1] - i[3]*(i[2] - 1)for i in zip(N_input, P, N, S)]
model.add(Conv2D(filters=6, kernel_size=(F[0],F[0]), padding='same', strides=S[0],
activation='relu', input_shape=input_shape))
model.add(MaxPooling2D(pool_size=F[1], strides=S[1]))
model.add(Conv2D(filters=16, kernel_size=(F[2],F[2]), padding='valid', strides=S[2],
activation='relu'))
model.add(MaxPooling2D(pool_size=F[3], strides=S[3]))
model.add(Conv2D(filters=120, kernel_size=(F[4],F[4]), padding='valid', strides=S[4],
activation='relu'))
model.add(Dropout(dropout))
model.add(Flatten())
 model.add(Dense(84, activation='relu'))
model.add(Dense(10, activation='softmax'))
if __name__ == '__main__' and run_model1:
model = Sequential()
build_lenet5(model, input_shape=X_train.shape[1:], dropout=0)
 model.summary()
filename = 'ames_house_xgb_model.pkl'
pickle.dump(xgb, open(filename, 'wb'))
X_test.to_csv('df_test_processed.csv', index=False )<predict_on_test>
hist_dict = {}
if __name__ == '__main__' and run_model1:
adam = optimizers.Adam()
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam)
 hist_dict['run_model1'] = model.fit(X_train, Y_train_oh, batch_size=64, epochs=20,
shuffle=True, validation_split=0.2, verbose=2)
row_to_show = 42
data_for_prediction = X_test.iloc[[row_to_show]]
y_sample = np.exp(xgb.predict(data_for_prediction))
explainer = shap.TreeExplainer(xgb)
 shap_values = explainer.shap_values(data_for_prediction )<predict_on_test>
def model_predict(model):
print("Generating test predictions...")
 predictions = model.predict_classes(X_test, verbose=1)
 print("OK.")
 return predictions
def model_predict_val(model, set_check):
print("Generating set predictions...")
 predictions = model.predict_classes(set_check, verbose=1)
 print("OK.")
 return predictions
def write_preds(preds, filename):
pd.DataFrame({"ImageId": list(range(1,len(preds)+1)) , "Label": preds} ).to_csv(filename, index=False, header=True)
 if __name__ == '__main__' and run_model1:
predictions = model_predict(model)
 print(predictions[:5])
 write_preds(predictions, "keras-lenet5-basic.csv")
data_for_prediction = X_test
y_sample = np.exp(xgb.predict(data_for_prediction))
explainer = shap.TreeExplainer(xgb)
 shap_values = explainer.shap_values(data_for_prediction )<define_variables>
if __name__ == '__main__' and run_model2:
model = Sequential()
build_lenet5(model, input_shape=X_train.shape[1:], dropout=0.3)
 model.summary()
adam = optimizers.Adam()
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam)
 hist_dict['run_model2'] = model.fit(X_train, Y_train_oh, batch_size=64, epochs=20, shuffle=True, validation_split=0.2, verbose=2 )
BATCH_SIZE = 128
EPOCHS = 15<load_from_csv>
if __name__ == '__main__' and run_model2:
predictions = model_predict(model)
 print(predictions[:5])
 write_preds(predictions, "keras-lenet5-basic-droupout.csv" )
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<set_options>

 datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=15,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
sns.set_theme(rc = {'grid.linewidth': 0.5,
'axes.linewidth': 0.75, 'axes.facecolor': '
'figure.facecolor': '
'xtick.labelcolor': '<prepare_x_and_y>
for x_batch, y_batch in datagen.flow(X_train, Y_train_oh, batch_size=9, shuffle = False):
print(x_batch.shape)
 print(y_batch.shape)
 break
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice', 'GarageArea', 'TotRmsAbvGrd'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))<create_dataframe>
if __name__ == '__main__' and run_model3:
X_train_s, X_val, Y_train_s, Y_val = train_test_split(X_train, Y_train_oh, test_size=0.13, random_state=42)
 model = Sequential()
build_lenet5(model, input_shape=X_train_s.shape[1:], dropout=0.15)
 model.summary()
adam = optimizers.Adam()
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam)
 epochs = 45
batch_size = 72
Train_gen_batch = datagen.flow(X_train_s, Y_train_s, batch_size=batch_size)
 datagen_no_aug = ImageDataGenerator()
Val_gen_batch = datagen_no_aug.flow(X_val, Y_val, batch_size=batch_size)
 hist_dict['run_model3'] = model.fit_generator(Train_gen_batch, epochs = epochs, verbose = 2,
steps_per_epoch = X_train.shape[0] // batch_size,
validation_data = Val_gen_batch,
validation_steps = X_val.shape[0] // batch_size,
callbacks=[learning_rate_reduction])
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )<data_type_conversions>
if __name__ == '__main__' and run_model3:
predictions = model_predict(model)
 print(predictions[:5])
 write_preds(predictions, "keras-lenet5-aug.csv")
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )<data_type_conversions>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )<data_type_conversions>
def build_net_advanced(model, input_shape=X_train.shape[1:], dropout=0.25):
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='same', strides=1,
activation='relu', input_shape=input_shape))
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='valid', strides=2,
activation='relu'))
model.add(MaxPooling2D(pool_size=(3,3), strides=1))
model.add(Dropout(dropout))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='same', strides=1,
activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='valid', strides=2,
activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=1))
model.add(Dropout(dropout))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(dropout))
model.add(Dense(128, activation='relu'))
model.add(Dropout(dropout))
model.add(Dense(10, activation='softmax'))
all_data["Alley"] = all_data["Alley"].fillna("None" )<data_type_conversions>
if __name__ == '__main__' and run_model_adv:
X_train_s, X_val, Y_train_s, Y_val = train_test_split(X_train, Y_train_oh, test_size=0.15, random_state=42)
 model = Sequential()
build_net_advanced(model, input_shape=X_train_s.shape[1:], dropout=0.3)
 model.summary()
adam = optimizers.Adam()
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam)
 epochs = 35
batch_size = 84
Train_gen_batch = datagen.flow(X_train_s, Y_train_s, batch_size=batch_size)
 datagen_no_aug = ImageDataGenerator()
Val_gen_batch = datagen_no_aug.flow(X_val, Y_val, batch_size=batch_size)
 hist_dict['run_model_adv'] = model.fit_generator(Train_gen_batch, epochs = epochs, verbose = 2,
steps_per_epoch = X_train.shape[0] // batch_size,
validation_data = Val_gen_batch,
validation_steps = X_val.shape[0] // batch_size,
callbacks=[learning_rate_reduction] )
all_data["Fence"] = all_data["Fence"].fillna("None" )<data_type_conversions>
if __name__ == '__main__' and run_model_adv:
predictions = model_predict(model)
 print(predictions[:5])
 write_preds(predictions, "keras-adv-net.csv")
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )<categorify>
_, X_val_check, _, Y_val_check = train_test_split(X_train, Y_train, test_size=0.1, random_state=1)
 Ypred_val_check = model_predict_val(model, set_check=X_val_check)
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))<data_type_conversions>
cm = confusion_matrix(Y_val_check.values, Ypred_val_check)
 cm
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )<data_type_conversions>
X_val_check[Ypred_val_check != Y_val_check.values].shape[0] / X_val_check.shape[0]
for col in('GarageYrBlt', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )<data_type_conversions>
display_digits(dim =(2,3),
X = X_val_check[Ypred_val_check != Y_val_check.values],
Y_true = Y_val_check.values[Ypred_val_check != Y_val_check.values],
pred = Ypred_val_check[Ypred_val_check != Y_val_check.values] )