Kaggle competition: Digit Recognizer
def impute(df, verbose):
if verbose:
print("
Missing values before imputing:")
 sums_df = pd.DataFrame(df.isnull().sum() , columns = ['Missing Values'])
 print(sums_df[sums_df['Missing Values'] > 0])
 total_missing_values = df.isnull().values.sum()
print("Total missing values: ", total_missing_values)
 df['MSZoning'] = df.groupby("Neighborhood")['MSZoning'].transform(lambda x: x.fillna(x.mode()))
df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode() [0])
 df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode() [0])
 df['SaleType'] = df['SaleType'].fillna(df['SaleType'].mode() [0])
 for name in df.select_dtypes("number"):
df[name] = df[name].fillna(0)
 for name in df.select_dtypes("category"):
df[name] = df[name].fillna("None")
 if verbose:
print("
missing values imputed: ", total_missing_values,)
 print("missing values remaining after imputing: ", df.isnull().values.sum())
 return df<categorify>
%matplotlib inline
df_train, df_test = preprocess_data(verbose=False )<compute_train_metric>
def Extract(file,labels=False):
	print("Beginning the extraction process...")
 	with open(file,'r')as f:
		data = f.read()
		data = data.split('
')
 	
	Data = []
	label = []
	
	dataLen = len(data)
 	
	
	print("preparing data into {} samples.. ".format(dataLen))
	
	for i in range(1,dataLen):
		if data[i] != '':
			if labels == False:
				Data.append(np.array(data[i].split(','), dtype=np.float32))
			else:
				Data.append(np.array(data[i].split(',')[1:], dtype=np.float32))
				label.append(np.array(data[i].split(',')[0], dtype=np.int8))
			
	Data = np.array(Data)/255
	del data
	if labels == False:
		return Data
	else:
		return(Data,label )
def score_dataset(X, y, model=XGBRegressor()):
for colname in X.select_dtypes(["category"]):
X[colname] = X[colname].cat.codes
log_y = np.log(y)
 score = cross_val_score(
 model, X, log_y, cv=5, scoring="neg_mean_squared_error")
 score = -1 * score.mean()
score = np.sqrt(score)
 return score<compute_test_metric>
( x_train,y_train),(x_test,y_test)= load_data()
data = Extract('/kaggle/input/digit-recognizer/test.csv')
(moreTrain,label)= Extract('/kaggle/input/digit-recognizer/train.csv',labels=True )
X = df_train.copy()
y = X.pop("SalePrice")
 baseline_score = score_dataset(X, y)
 print(f"Baseline score: {baseline_score:.5f} RMSLE" )<prepare_x_and_y>
y_test = to_categorical(y_test)
 x_train = x_train.astype(np.float32)/ 255
x_test = x_test.astype(np.float32)/ 255
x_test = x_test.reshape(len(x_test),28,28,1)
 moreTrain /= 255
print(x_train.shape)
 print(y_train.shape)
 x_train = np.concatenate(( x_train,moreTrain.reshape(len(moreTrain),28,28)) ,axis=0)
 y_train = np.concatenate(( y_train,label),axis=0)
 x_train = x_train.reshape(len(x_train),28,28,1)
 y_train = to_categorical(y_train)
 print(x_train.shape)
 print(y_train.shape )
X = df_train.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 mi_scores<drop_column>
model = Sequential()
model.add(Conv2D(filters=16, kernel_size=5, strides=1, padding='same', activation='relu', input_shape=(28, 28, 1), name='conv1'))
model.add(Conv2D(filters=16, kernel_size=5, strides=1, padding='same', activation='relu', input_shape=(28, 28, 1), name='conv2'))
model.add(MaxPool2D(pool_size=(2, 2),name='pool1'))
model.add(Conv2D(filters=32, kernel_size=5, strides=1,padding='same', activation='relu',name='conv3'))
model.add(Conv2D(filters=32, kernel_size=3, strides=1,padding='same', activation='relu',name='conv4'))
model.add(MaxPool2D(pool_size=(2, 2),name='pool2'))
model.add(Conv2D(filters=64, kernel_size=3, strides=1,padding='same', activation='relu',name='conv5'))
model.add(Conv2D(filters=64, kernel_size=3, strides=1,padding='valid', activation='relu',name='conv6'))
model.add(Flatten(name='flat'))
model.add(Dense(512,activation='relu',name='dense1'))
model.add(Dropout(0.2,name='dropout'))
model.add(Dense(10,activation='softmax',name='res'))
model.summary()
def drop_uninformative(df, mi_scores, verbose=False):
if verbose:
print('Dropping the following features with low mi_scores:')
 print(df.loc[:, mi_scores == 0.0].columns)
 return df.loc[:, mi_scores > 0.0]<create_dataframe>
class Callback(ks.callbacks.Callback):
def on_epoch_end(self,epoch,logs={}):
if(logs['val_acc'] >= 0.999):
print("
Valication accuracy reached more than 99.9%, training stopped")
 self.model.stop_training=True
X = df_train.copy()
y = X.pop("SalePrice")
 X = drop_uninformative(X, mi_scores, verbose=True)
 score_dataset(X, y )<categorify>
earlyStop = Callback()
adam = ks.optimizers.Adam(lr=1e-3)
 model.compile(loss='categorical_crossentropy',optimizer=adam, metrics=['accuracy'])
 checkpoint = ks.callbacks.ModelCheckpoint('model.h5',save_best_only=True, monitor='val_acc', mode='max' )
def label_encode_these(df, cols=[], verbose=False):
X = df.copy()
for colname in cols:
X[colname] = X[colname].cat.codes
if verbose:
print(len(cols), "categorical features label encoded:",)
 print_n_per_line(cols, 5)
 return X 
<categorify>
history = model.fit(x_train, y_train, validation_data=(x_test,y_test),batch_size=1024, epochs=100, shuffle=True, verbose=1, callbacks=[checkpoint,earlyStop] )
def one_hot_encode_except(df, label_encoded_cols=[], verbose=False):
X = df.copy()
remaining_col_names = []
for col_name in list(X):
if X[col_name].dtype.name == 'category' and col_name not in label_encoded_cols:
remaining_col_names.append(col_name)
 X = pd.get_dummies(X, columns=remaining_col_names)
 if verbose:
print("One Hot encoding applied to remaining", len(remaining_col_names), "categorical features:")
 print_n_per_line(remaining_col_names, 5)
 return X<feature_engineering>
print(model.evaluate(x_test,y_test))
print(modelBest.evaluate(x_test,y_test))
def mathematical_transforms(df, verbose=False):
X = pd.DataFrame()
X["LivLotRatio"] = df.GrLivArea / df.LotArea
X["Spaciousness"] =(df.FirstFlrSF + df.SecondFlrSF)/ df.TotRmsAbvGrd
X["TotalSF"] = df.TotalBsmtSF + df.FirstFlrSF + df.SecondFlrSF
new_features = ["LivLotRatio", "Spaciousness", "TotalSF"]
if verbose and new_features:
print(len(new_features), "new features created via mathematical transforms: ")
 print(" ", new_features)
 return X
def interactions(df, verbose=False):
new_interactions = ["Bldg with GrLivArea"]
columns_to_interact_with_GrLivArea = ['BldgType']
new_prefixes = ['Bldg']
X = pd.get_dummies(df[columns_to_interact_with_GrLivArea], columns=columns_to_interact_with_GrLivArea, prefix=new_prefixes)
 X = X.mul(df.GrLivArea, axis=0)
 if verbose and new_interactions:
print(len(new_interactions), "new interaction features created: ")
 print(" ", new_interactions, " :")
 for prefix in new_prefixes:
print_n_per_line(list(X.columns[X.columns.str.startswith(prefix)]), 5)
 return X
def counts(df, verbose=False):
X = pd.DataFrame()
X["PorchTypes"] = df[[
"WoodDeckSF",
"OpenPorchSF",
"EnclosedPorch",
"ThreeSeaPorch",
"ScreenPorch",
]].gt(0.0 ).sum(axis=1)
 new_counted_features = ["PorchTypes"]
if verbose and new_counted_features:
print(len(new_counted_features), 'new "feature count" type features created: ')
 print(" ", new_counted_features)
 return X
def break_down(df, verbose=False):
X = pd.DataFrame()
X["MSClass"] = df.MSSubClass.str.split("_", n=1, expand=True)[0]
return X
def group_transforms(df, verbose=False):
X = pd.DataFrame()
X["MedNeighLivArea"] = df.groupby("Neighborhood")["GrLivArea"].transform("median")
 X["MedNeighLotArea"] = df.groupby("Neighborhood")["LotArea"].transform("median")
 X["MedQualLivArea"] = df.groupby("OverallQual")["GrLivArea"].transform("median")
 new_transforms = ["MedNeighLivArea", "MedNeighLotArea", "MedQualLivArea"]
if verbose and new_transforms:
print(len(new_transforms), "new features created via groupby transform of a feature: ")
 print(" ", new_transforms)
 return X<create_dataframe>
with open('result.csv','w')as f:
f.write('ImageId,Label
' )
<feature_engineering><EOS> 
i=0
for x in data:
with open('result.csv','a')as f:
pred = modelBest.predict(x.reshape(1,28,28,1))
val = int(np.argmax(pred,axis=1))
i += 1
res = str(i)+ ',' + str(val)+ '
'
f.write(res )