Kaggle competition: Digit Recognizer
store = pd.read_csv(r'.. /input/rossmann-store-sales/store.csv', index_col='Store')
 store.head()<data_type_conversions>
print(tf.__version__)
mean_dist = store['CompetitionDistance'].mean()
store.loc[store['CompetitionDistance'].isnull() , 'CompetitionDistance'] = mean_dist
store.loc[store['Promo2'] == 0, ['Promo2SinceWeek','Promo2SinceYear']] = 0
store['Promo2SinceWeek'] = store['Promo2SinceWeek'].astype('int')
 store['Promo2SinceYear'] = store['Promo2SinceYear'].astype('int')
 store['CompetitionOpen'] = 1
store.loc[store['CompetitionOpenSinceMonth'].isnull() ,['CompetitionOpen', 'CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear']] = 0
store['CompetitionOpenSinceMonth'] = store['CompetitionOpenSinceMonth'].astype('int')
 store['CompetitionOpenSinceYear'] = store['CompetitionOpenSinceYear'].astype('int')
 label_map = {'PromoInterval' : {'Jan,Apr,Jul,Oct' : 1,
'Feb,May,Aug,Nov' : 2,
'Mar,Jun,Sept,Dec' : 3,
np.nan : 0},
'Assortment' : {'a':0,
'b':1,
'c':2}}
store.replace(label_map, inplace=True )<feature_engineering>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_df.head()
train.loc[train['StateHoliday'] != '0', 'StateHoliday'] = '1'
train['StateHoliday'] = train['StateHoliday'].astype('int')
 test.loc[test['StateHoliday'] != '0', 'StateHoliday'] = '1'
test['StateHoliday'] = test['StateHoliday'].astype('int')
 train['year'] = train['Date'].dt.year
train['month'] = train['Date'].dt.month
train['day'] = train['Date'].dt.day
test['year'] = test['Date'].dt.year
test['month'] = test['Date'].dt.month
test['day'] = test['Date'].dt.day
test.loc[test['Open'].isnull() , 'Open'] = 1
test['Open'] = test['Open'].astype('int')
 train.drop('Customers', axis=1, inplace=True )<choose_model_class>
y_train = train_df.label.values
x_train = train_df.drop(columns=["label"] ).values
x_test = test_df.values
x_train[:10]
kfold = KFold(n_splits=5, random_state=2021, shuffle=True)
 parameters = {'learning_rate' : [0.1,0.2,0.35]}
clf = XGBRegressor(random_state=2021, use_label_encoder=False, n_estimators=100, max_depth=4 )<create_dataframe>
x_train = x_train / 255.0
x_test = x_test / 255.0
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(784,)) ,
tf.keras.layers.Dense(256, activation=tf.nn.relu),
tf.keras.layers.Dense(128, activation=tf.nn.relu),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
 ])
submit_frame = pd.DataFrame(columns=['Id','Sales'] )<feature_engineering>
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('acc')> 0.997):
print("
Reached 99% accuracy so cancelling training!")
 self.model.stop_training = True
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy'])
 model.fit(x_train, y_train, epochs=30)
for st_no in range(1,1116):
train_t = train.loc[train['Store'] == st_no].copy()
train_t.drop('Store', axis=1, inplace=True)
 test_t = test.loc[test['Store'] == st_no].copy()
test_t.drop('Store',axis=1,inplace=True)
 st_t = store.loc[store.index==st_no].iloc[0,:]
if test_t.shape[0] > 0:
train_t['Promo2'] = 0
train_t['NewPromoAge'] = 0
train_t['Competition'] = 0
test_t['Promo2'] = 0
test_t['NewPromoAge'] = 0
test_t['Competition'] = 0
if st_t['Promo2'] == 1:
train_t['Promo2'], train_t['NewPromoAge'] = zip(*train_t['Date'].map(calc_promo))
test_t['Promo2'], test_t['NewPromoAge'] = zip(*test_t['Date'].map(calc_promo))
if st_t['CompetitionOpen']:
train_t['Competition'] = train_t['Date'].map(calc_comp)
 test_t['Competition'] = test_t['Date'].map(calc_comp)
 train_t.drop('Date', axis=1, inplace=True)
 test_t.drop('Date', axis=1, inplace=True)
 X_train_t = train_t.drop('Sales',axis=1)
 y_train_t = train_t['Sales']
cv = GridSearchCV(clf,
param_grid=parameters,
cv=kfold,
scoring='neg_mean_squared_error')
 cv.fit(X_train_t, y_train_t)
 y_pred_t = cv.predict(test_t)
 y_pred_t[y_pred_t < 0] = 0
out_frame = pd.DataFrame([test_t.index, y_pred_t] ).T
out_frame.columns = ['Id','Sales']
out_frame['Id'] = out_frame['Id'].astype('int')
 submit_frame = submit_frame.append(out_frame)
 print(f'Predicted on : {st_no}, train_rows={train_t.shape[0]}, test_rows={len(y_pred_t)}')
 else:
print(f'Skipped: {st_no}, train_rows={train_t.shape[0]}' )<save_to_csv>
classifications = model.predict(x_test )
submit_frame.sort_values('Id', inplace=True)
 print(submit_frame.shape)
 print(submit_frame.head())
 submit_frame.to_csv(r'submission.csv', index=False )<set_options>
def write_submissions(file_name, imageId, predictions):
output = pd.DataFrame({
'ImageId': imageId, 'Label': predictions
})
 output.to_csv(file_name, index=False)
 write_submissions('submission_1.csv', pd.Series(range(1,28001)) , np.argmax(classifications, axis=1))
warnings.filterwarnings('ignore' )<load_from_csv>
y_train = train_df.label.values
x_train = train_df.drop(columns=["label"] ).values
x_test = test_df.values
df_train = pd.read_csv('.. /input/rossmann-store-sales/train.csv' )<count_missing_values>
x_train = x_train.reshape(42000, 28, 28, 1)
 x_test = x_test.reshape(28000, 28, 28, 1)
 x_train = x_train / 255.0
x_test = x_test / 255.0
df_train.isnull().sum()<count_values>
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(64,(3, 3), activation='relu', input_shape=(28, 28, 1)) ,
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Conv2D(64,(3, 3), activation='relu'),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(1024, activation='relu'),
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 model.summary()
df_train['DayOfWeek'].value_counts()<count_unique_values>
model.fit(x_train, y_train, epochs=20 )
len(df_train['Store'].unique() )<count_values>
classifications = model.predict(x_test)
 write_submissions('submission_2.csv', pd.Series(range(1,28001)) , np.argmax(classifications, axis=1))
<count_values><EOS> 
f, axarr = plt.subplots(3, 4)
 FIRST_IMAGE = 0
SECOND_IMAGE = 7
THIRD_IMAGE = 8
CONVOLUTION_NUMBER = 1
layer_outputs = [layer.output for layer in model.layers]
activation_model = tf.keras.models.Model(inputs=model.input, outputs=layer_outputs)
 for x in range(0, 4):
f1 = activation_model.predict(x_train[FIRST_IMAGE].reshape(1, 28, 28, 1)) [x]
axarr[0, x].imshow(f1[0, :, :, CONVOLUTION_NUMBER], cmap='inferno')
 axarr[0, x].grid(False)
 f2 = activation_model.predict(x_train[SECOND_IMAGE].reshape(1, 28, 28, 1)) [x]
axarr[1, x].imshow(f2[0, :, :, CONVOLUTION_NUMBER], cmap='inferno')
 axarr[1, x].grid(False)
 f3 = activation_model.predict(x_train[THIRD_IMAGE].reshape(1, 28, 28, 1)) [x]
axarr[2, x].imshow(f3[0, :, :, CONVOLUTION_NUMBER], cmap='inferno')
 axarr[2, x].grid(False)