Kaggle competition: Digit Recognizer
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
<load_from_csv>
data_dir='/kaggle/input/digit-recognizer/'
<rename_columns>
train=pd.read_csv(data_dir+'train.csv')
 test=pd.read_csv(data_dir+'test.csv' )
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<merge>
y_train=train['label']
x_train=train.drop('label',axis=1 )
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
<drop_column>
def image_printer(i,df):
idx=i
data=df.iloc[idx].to_numpy().reshape(28,28 ).astype('uint8')
 plt.imshow(data )
del content_explation_agg<data_type_conversions>
x_test=test
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )<data_type_conversions>
import tensorflow as tf
import keras
from keras import backend as k
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Activation, BatchNormalization
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import plot_model
from keras.callbacks import EarlyStopping
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )<drop_column>
img_cols=28
img_rows=28
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()<categorify>
if k.image_data_format=='channels_first':
x_train=x_train.values.reshape(x_train.shape[0],1,img_rows,img_cols)
 test=test.values.reshape(test.shape[0],1,img_rows,img_cols)
 x_train=x_train/255.0
test=test/255.0
input_shape=(1,img_rows,img_cols)
 else:
x_train=x_train.values.reshape(x_train.shape[0],img_rows,img_cols,1)
 test=test.values.reshape(test.shape[0],img_rows,img_cols,1)
 x_train=x_train/255.0
test=test/255.0
input_shape=(img_rows,img_cols,1)
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<data_type_conversions>
earlystopping=EarlyStopping(monitor='val_accuracy',mode='auto',patience=10,restore_best_weights=True)
 modelacc=[]
nfilters=[64,128,256]
conv_layers=[1,2,3,4,5]
dense_layers=[0,1,2,3,4]
dropouts=[0.5]
for filters in nfilters:
for conl in conv_layers:
for densel in dense_layers:
for dp in dropouts:
cnnsays='Feature Maps: {} Convlayers: {} Denselayers: {} Dropouts: {}'.format(filters,conl,densel,dp)
 print(cnnsays)
 model=Sequential()
model.add(Conv2D(filters,(3,3),input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(dp))
for i in range(conl-1):
model.add(Conv2D(filters,(3,3),padding='same'))
model.add(BatchNormalization())
 model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2,2),padding='same'))
model.add(Dropout(dp))
model.add(Flatten())
 for i in range(densel):
model.add(Dense(256))
model.add(BatchNormalization())
 model.add(Activation("relu"))
model.add(Dropout(dp))
model.add(Dense(10,activation='softmax'))
model.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 EPOCHS=30
history=model.fit(x_train,y_train,batch_size=32,epochs=EPOCHS,validation_split=0.2,callbacks=[earlystopping])
 modelacc.append([round(100*max(history.history['val_accuracy']),2),cnnsays])
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )<categorify>
print('Highest validation accuracy {}'.format(round(100*max(history.history['val_accuracy']),2)) )
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
<data_type_conversions>
modelacc.sort(reverse=True)
 modelacc
<drop_column>
pred=model.predict([test])
 soln=[]
for i in range(len(pred)) :
soln.append(np.argmax(pred[i]))
<define_variables><EOS> 
final=pd.DataFrame()
final['ImageId']=[i+1 for i in x_test.index]
final['Label']=soln
final.to_csv('newmnistcnn',index=False )