Kaggle competition: Digit Recognizer
def score_dataset(X,y,model = XGBRegressor()):
for colname in X.select_dtypes(['category']):
X[colname] = X[colname].cat.codes
log_y = np.log(y)
 score = cross_val_score(model, X, log_y, cv=5, scoring = 'neg_mean_squared_error')
 score = -1 * score.mean()
score = np.sqrt(score)
 return score 
<create_dataframe>
from keras.layers import *
from keras.models import Model
X = df_train.copy()
y = X.pop('SalePrice')
 baseline_score = score_dataset(X,y)
 print(baseline_score )<statistical_test>
def normalize(x):

 return x /(K.sqrt(K.mean(K.square(x)))+ K.epsilon())
 def deprocess_image(x):

 x -= x.mean()
x /=(x.std() + K.epsilon())
 x *= 0.25
x += 0.5
x = np.clip(x, 0, 1)
 x *= 255
if K.image_data_format() == 'channels_first':
x = x.transpose(( 1, 2, 0))
x = np.clip(x, 0, 255 ).astype('uint8')
 return x
def process_image(x, former):

 if K.image_data_format() == 'channels_first':
x = x.transpose(( 2, 0, 1))
return(x / 255 - 0.5)* 4 * former.std() + former.mean()
def visualize_layer(model,
layer_name,
step=1.,
epochs=15,
upscaling_steps=9,
upscaling_factor=1.2,
output_dim=(412, 412),
filter_range=(0, None)) :

 def _generate_filter_image(input_img,
layer_output,
filter_index):

 s_time = time.time()
if K.image_data_format() == 'channels_first':
loss = K.mean(layer_output[:, filter_index, :, :])
 else:
loss = K.mean(layer_output[:, :, :, filter_index])
 grads = K.gradients(loss, input_img)[0]
grads = normalize(grads)
 iterate = K.function([input_img], [loss, grads])
 intermediate_dim = tuple(
 int(x /(upscaling_factor ** upscaling_steps)) for x in output_dim)
 if K.image_data_format() == 'channels_first':
input_img_data = np.random.random(
(1, 3, intermediate_dim[0], intermediate_dim[1]))
else:
input_img_data = np.random.random(
(1, intermediate_dim[0], intermediate_dim[1], 3))
input_img_data =(input_img_data - 0.5)* 20 + 128
for up in reversed(range(upscaling_steps)) :
for _ in range(epochs):
loss_value, grads_value = iterate([input_img_data])
 input_img_data += grads_value * step
if loss_value <= K.epsilon() :
return None
intermediate_dim = tuple(
 int(x /(upscaling_factor ** up)) for x in output_dim)
 img = deprocess_image(input_img_data[0])
 img = np.array(pil_image.fromarray(img ).resize(intermediate_dim,
pil_image.BICUBIC))
input_img_data = np.expand_dims(
 process_image(img, input_img_data[0]), 0)
 img = deprocess_image(input_img_data[0])
 e_time = time.time()
print('Costs of filter {:3}: {:5.0f}({:4.2f}s)'.format(filter_index,
loss_value,
e_time - s_time))
return img, loss_value
def _draw_filters(filters, n=None):

 if n is None:
n = int(np.floor(np.sqrt(len(filters))))
filters.sort(key=lambda x: x[1], reverse=True)
 filters = filters[:n * n]
MARGIN = 5
width = n * output_dim[0] +(n - 1)* MARGIN
height = n * output_dim[1] +(n - 1)* MARGIN
stitched_filters = np.zeros(( width, height, 3), dtype='uint8')
 for i in range(n):
for j in range(n):
img, _ = filters[i * n + j]
width_margin =(output_dim[0] + MARGIN)* i
height_margin =(output_dim[1] + MARGIN)* j
stitched_filters[
width_margin: width_margin + output_dim[0],
height_margin: height_margin + output_dim[1], :] = img
save_img('vgg_{0:}_{1:}x{1:}.png'.format(layer_name, n), stitched_filters)
 assert len(model.inputs)== 1
input_img = model.inputs[0]
layer_dict = dict([(layer.name, layer)for layer in model.layers[1:]])
 output_layer = layer_dict[layer_name]
assert isinstance(output_layer, layers.Conv2D)
 filter_lower = filter_range[0]
filter_upper =(filter_range[1]
if filter_range[1] is not None
else len(output_layer.get_weights() [1]))
assert(filter_lower >= 0
and filter_upper <= len(output_layer.get_weights() [1])
 and filter_upper > filter_lower)
 print('Compute filters {:} to {:}'.format(filter_lower, filter_upper))
processed_filters = []
for f in range(filter_lower, filter_upper):
img_loss = _generate_filter_image(input_img, output_layer.output, f)
 if img_loss is not None:
processed_filters.append(img_loss)
 print('{} filter processed.'.format(len(processed_filters)))
 _draw_filters(processed_filters)
def make_mi_scores(X, y):
X = X.copy()
for colname in X.select_dtypes(['object','category']):
X[colname], _ = X[colname].factorize()
discrete_features = [pd.api.types.is_integer_dtype(t)for t in X.dtypes]
mi_scores = mutual_info_regression(X, y, discrete_features = discrete_features, random_state=0)
 mi_scores = pd.Series(mi_scores, name = 'MI Scores', index = X.columns)
 mi_scores = mi_scores.sort_values(ascending = False)
 return mi_scores 
<prepare_x_and_y>
train_data=pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test_data=pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
X = df_train.copy()
y = X.pop('SalePrice')
 mi_scores = make_mi_scores(X,y)
 mi_scores<drop_column>
y=train_data["label"]
X=train_data.copy()
del X["label"]
def drop_uninformative(df, mi_scores):
return df.loc[:, mi_scores>0.0] 
<create_dataframe>
SIZE=32
X = df_train.copy()
y = X.pop('SalePrice')
 X = drop_uninformative(X, mi_scores)
 score_dataset(X,y )<categorify>
def reshape32(img):
img=img.reshape(( 28,28))
img=np.pad(img,(( SIZE-28)//2,(SIZE-28)//2))
img=img.reshape(( SIZE,SIZE,1))
return img
def label_encode(df):
X = df.copy()
for colname in X.select_dtypes(['category']):
X[colname] = X[colname].cat.codes
return X 
<feature_engineering>
new_X=[]
for i,img in enumerate(X.values):
new_X.append(reshape32(img))
new_X=np.array(new_X)
 new_X[new_X<50]=0
def mathematical_transforms(df):
X = pd.DataFrame()
X['LivLotRatio'] = df.GrLivArea / df.LotArea
X['Spaciousness'] =(df.FirstFlrSF + df.SecondFlrSF)/ df.TotRmsAbvGrd
X['Feet'] = np.sqrt(df.GrLivArea)
 X['TotalSF'] = df.TotalBsmtSF + df.FirstFlrSF + df.SecondFlrSF
X['TotalBathrooms'] = df.FullBath + 0.5* df.HalfBath + df.BsmtFullBath + 0.5 * df.BsmtHalfBath
X['TotalPorchSF']= df.OpenPorchSF + df.Threeseasonporch + df.EnclosedPorch + df.ScreenPorch + df.WoodDeckSF
return X
def interactions(df):
X = pd.get_dummies(df.BldgType, prefix = 'Bldg')
 X = X.mul(df.GrLivArea, axis = 0)
 return X
def counts(df):
X = pd.DataFrame()
X['PorchTypes'] = df[[
'WoodDeckSF',
'OpenPorchSF',
'EnclosedPorch',
'Threeseasonporch',
'ScreenPorch',
]].gt(0.0 ).sum(axis = 1)
 return X
def group_transforms(df):
X = pd.DataFrame()
X['MedNhbdArea'] = df.groupby('Neighborhood')['GrLivArea'].transform('median')
 return X
def break_down(df):
X = pd.DataFrame()
X['MSClass'] = df.MSSubClass.str.split('_', n=1, expand = True)[0]
return X 
<create_dataframe>
train_X,val_X,train_y,val_y = train_test_split(new_X/255,y,test_size=0.1 )
def apply_pca(X, standarize = True):
if standarize:
X =(X - X.mean(axis=0)) / X.std(axis=0)
 pca = PCA()
X_pca = pca.fit_transform(X)
 component_names = [f'PC{i+1}' for i in range(X_pca.shape[1])]
X_pca = pd.DataFrame(X_pca, columns = component_names)
 loadings = pd.DataFrame(pca.components_.T, columns = component_names, index = X.columns)
 return pca, X_pca, loadings
def pca_inspired(df):
X = pd.DataFrame()
X['Feature1'] = df.GrLivArea + df.TotalBsmtSF
X['Feature2'] = df.YearRemodAdd * df.TotalBsmtSF
return X
def pca_components(df, features):
X = df.loc[:, features]
_, X_pca, _ = apply_pca(X)
 return X_pca
pca_features = ['GarageArea', 'YearRemodAdd','TotalBsmtSF','GrLivArea'] 
<create_dataframe>
inp=Input(shape=(32,32,1))
model = Conv2D(filters=32, kernel_size=(2, 2), padding='SAME', activation='relu',name="conv32" )(inp)
 model = Conv2D(filters=32, kernel_size=(2, 2), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=32, kernel_size=(2, 2), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu' )(model)
 model = Dropout(rate=0.3 )(model)
 model = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu',name="conv64" )(model)
 model = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=64, kernel_size=(5, 5), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.2 )(model)
 model = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu',name="conv128" )(model)
 model = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Conv2D(filters=128, kernel_size=(7, 7), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = Dropout(rate=0.2 )(model)
 model = Conv2D(filters=256, kernel_size=(15, 15), padding='SAME', activation='relu',name="conv256" )(model)
 model = Conv2D(filters=256, kernel_size=(15, 15), padding='SAME', activation='relu' )(model)
 model = Conv2D(filters=256, kernel_size=(15, 15), padding='SAME', activation='relu' )(model)
 model = BatchNormalization(momentum=0.15 )(model)
 model = MaxPool2D(pool_size=(2, 2))(model)
 model = Dropout(rate=0.2 )(model)
 my_x=Flatten()(model)
 my_x=Dense(256,activation='relu',kernel_initializer='he_uniform',bias_initializer='zeros' )(my_x)
 my_x=Dropout(0.2 )(my_x)
 my_x=Dense(128,activation='relu',kernel_initializer='he_uniform',bias_initializer='zeros' )(my_x)
 my_x=Dropout(0.2 )(my_x)
 my_x=Dense(64,activation='relu',name='my' )(my_x)
 my_x=Dropout(0.2 )(my_x)
 my_x=Dense(32,activation='relu',name='my2' )(my_x)
 my_x=Dropout(0.2 )(my_x)
 preds=Dense(10,activation='softmax',kernel_initializer='he_uniform',bias_initializer='zeros',name='output' )(my_x)
 my_model=Model(inputs=inp,outputs=preds)
def indicate_outliers(df):
X_new = pd.DataFrame()
X_new['Outlier'] =(df.Neighborhood == 'Edwards')&(df.SaleCondition == 'Partial')
 return X_new 
<categorify>
my_model.compile(optimizer=Adadelta() ,loss='categorical_crossentropy',metrics=['accuracy','mse'] )
class CrossFoldEncoder:
def __init__(self,encoder, **kwargs):
self.encoder_ = encoder
self.kwargs_ = kwargs
self.cv_ = KFold(n_splits = 5)
 def fit_transform(self,X,y,cols):
self.fitted_encoders_ = []
self.cols_ = cols
X_encoded = []
for idx_encode, idx_train in self.cv_.split(X):
fitted_encoder = self.encoder_(cols = cols, **self.kwargs_)
 fitted_encoder.fit(X.iloc[idx_encode, :], y.iloc[idx_encode])
 X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])
 self.fitted_encoders_.append(fitted_encoder)
 X_encoded = pd.concat(X_encoded)
 X_encoded.columns = [name + '_encoded' for name in X_encoded.columns]
return X_encoded
def transform(self,X):
X_encoded_list = []
for fitted_encoder in self.fitted_encoders_:
X_encoded = fitted_encoder.transform(X)
 X_encoded_list.append(X_encoded[self.cols_])
 X_encoded = reduce(lambda x,y : x.add(y,fill_value=0), X_encoded_list)/ len(X_encoded_list)
 X_encoded.columns = [name + '_encoded' for name in X_encoded.columns]
return X_encoded<merge>
rlrp = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, min_delta=1E-30,verbose=1)
 history=my_model.fit(x=train_X,y=pd.get_dummies(train_y),validation_data=(val_X,pd.get_dummies(val_y)) ,epochs=100, batch_size=1024,callbacks=[rlrp])
def create_features(df, df_test = None):
X = df.copy()
y = X.pop('SalePrice')
 mi_scores = make_mi_scores(X,y)
 if df_test is not None:
X_test = df_test.copy()
X_test.pop('SalePrice')
 X = pd.concat([X, X_test])
 X = drop_uninformative(X, mi_scores)
 X = X.join(mathematical_transforms(X))
X = X.join(interactions(X))
X = X.join(counts(X))
X = X.join(break_down(X))
X = X.join(group_transforms(X))
X = X.join(cluster_labels(X, cluster_features, n_clusters = 20))
X = X.join(pca_inspired(X))
X = label_encode(X)
 if df_test is not None:
X_test = X.loc[df_test.index, :]
X.drop(df_test.index, inplace=True)
 encoder = CrossFoldEncoder(MEstimateEncoder, m=1)
 X = X.join(encoder.fit_transform(X, y, cols= ['MSSubClass']))
if df_test is not None:
X_test = X_test.join(encoder.transform(X_test))
if df_test is not None:
return X, X_test
else:
return X
df_train, df_test = load_data()
X_train = create_features(df_train)
 y_train = df_train.loc[:, 'SalePrice']
score_dataset(X_train, y_train )<prepare_x_and_y>
for layer in my_model.layers:
print(layer.name,)
 if 'conv' not in layer.name:
continue
filters, biases = layer.get_weights()
filters, biases = layer.get_weights()
f_min, f_max = filters.min() , filters.max()
filters =(filters - f_min)/(f_max - f_min)
 n_filters, ix = 6, 1
for i in range(n_filters):
f = filters[:, :, :, i]
for j in range(1):
ax = plt.subplot(n_filters, 3, ix)
 ax.set_xticks([])
 ax.set_yticks([])
 plt.imshow(f[:, :, j], cmap='gray')
 ix += 1
plt.show()
X_train = create_features(df_train)
 y_train = df_train.loc[:, 'SalePrice']
xgb_params = dict(max_depth = 6,
learning_rate = 0.01,
n_estimators = 1000,
min_child_weight = 1,
colsample_bytree = 0.7,
subsample = 0.7,
reg_alpha = 0.5,
reg_lambda = 1,
num_parallel_tree = 1)
 xgb = XGBRegressor(**xgb_params)
 score_dataset(X_train, y_train, xgb )<find_best_params>
for i in range(len(val_X)) :
if np.argmax(my_model.predict(val_X[i].reshape(1,32,32,1)) ,axis=1)!=val_y.values[i]: 
(plt.imshow(val_X[i].reshape(32,32),))
plt.show()
print("Label : ",val_y.values[i])
 print("Prediction : ",np.argmax(my_model.predict(val_X[i].reshape(1,32,32,1)) ,axis=1))
def objective(trial):
xgb_params = dict(
 max_depth = trial.suggest_int('max_depth',2,10),
learning_rate = trial.suggest_float('learning_rate',1e-4, 1e-1, log=True),
n_estimators = trial.suggest_int('n_estimators',1000,8000),
min_child_weight = trial.suggest_int('min_child_weight', 1,10),
colsample_bytree = trial.suggest_float('colsample_bytree',0.2,1),
subsample = trial.suggest_float('subsample',0.2,1),
reg_alpha = trial.suggest_float('reg_alpha',1e-4, 1e2, log=True),
reg_lambda = trial.suggest_float('reg_lambda', 1e-4, 1e2, log=True))
xgb = XGBRegressor(**xgb_params)
 return score_dataset(X_train, y_train, xgb)
 study = optuna.create_study(direction = 'minimize')
 study.optimize(objective, n_trials = 20)
 xgb_params = study.best_params 
<save_to_csv>
test_X=[]
for i,img in enumerate(test_data.values):
z=reshape32(img)
 test_X.append(z)
 test_X=np.array(test_X)
 test_X[test_X<50]=0
test_X=test_X/255
<load_from_csv><EOS> 
sol=np.argmax(my_model.predict(( test_X)) ,axis=1)
 df=pd.DataFrame(sol)
 df.index+=1
df.to_csv("/kaggle/working/sol_final.csv",index=True,header=["Label"],index_label=["ImageId"] )