Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
import pandas as pd
import numpy as np
import tensorflow as tf
NUC_MODEL = ".. /input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth"
CELL_MODEL = ".. /input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth"
segmentator_even_faster = CellSegmentator(
 NUC_MODEL,
CELL_MODEL,
device="cuda",
multi_channel_model=True,
padding=True,
return_without_scale_restore=True
)<create_dataframe>
train=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 print("Training Dataset Shape:",train.shape)
 train.head()
yield_ims_1728 = Yield_Images_Dataset(predict_df_1728)
 yield_ims_2048 = Yield_Images_Dataset(predict_df_2048)
 yield_ims_3072 = Yield_Images_Dataset(predict_df_3072)
 yield_ims_4096 = Yield_Images_Dataset(predict_df_4096)
 dataloader_ims_seg_1728 = DataLoader(yield_ims_1728, batch_size=24,
shuffle=False, num_workers=0)
 dataloader_ims_seg_2048 = DataLoader(yield_ims_2048, batch_size=12,
shuffle=False, num_workers=0)
 dataloader_ims_seg_3072 = DataLoader(yield_ims_3072, batch_size=3,
shuffle=False, num_workers=0)
 dataloader_ims_seg_4096 = DataLoader(yield_ims_4096, batch_size=3,
shuffle=False, num_workers=0)
 dataloaders_all_sizes = [dataloader_ims_seg_1728, dataloader_ims_seg_2048, dataloader_ims_seg_3072, dataloader_ims_seg_4096]
start_time = time.time()
even_faster_outputs = []
output_ids = []
batch_size = 24
sizes_list = []
im_proc = 0
for i, dataloader_ims_seg in enumerate(dataloaders_all_sizes):
print(f"GETTING IMAGE SIZES: {IMAGE_SIZES[i]}, BATCHES: {len(dataloader_ims_seg)}")
 print
for blue_images, ryb_images, sizes, _ids in dataloader_ims_seg:
print(f"SEGMENT COUNT: {im_proc}")
 blue_batch = blue_images.numpy()
ryb_batch = ryb_images.numpy()
nuc_segmentations = segmentator_even_faster.pred_nuclei(blue_batch)
 cell_segmentations = segmentator_even_faster.pred_cells(ryb_batch, precombined=True)
 for data_id, nuc_seg, cell_seg, size in zip(_ids, nuc_segmentations, cell_segmentations, sizes):
_, cell = utils.label_cell(nuc_seg, cell_seg)
 even_faster_outputs.append(np.ubyte(cell))
output_ids.append(data_id)
 sizes_list.append(size.numpy())
 im_proc += len(_ids)
 del dataloader_ims_seg
print(time.time() - start_time)
 cell_masks_df = pd.DataFrame(list(zip(output_ids, even_faster_outputs,sizes_list)) ,
columns=["ID", "mask", "ori_size"] )<drop_column>
test=pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print('Test Dataset Shape:',test.shape)
 test.head()
cell_masks_df = cell_masks_df.set_index('ID')
 cell_masks_df = cell_masks_df.reindex(index=data_df['ID'])
 cell_masks_df = cell_masks_df.reset_index()<set_options>
x=x.reshape(-1,28,28,1)
 test=test.values.reshape(-1,28,28,1)
 x=x/255
test=test/255
y=to_categorical(y )
del sizes_list
del even_faster_outputs
del output_ids
del segmentator_even_faster
del yield_ims_1728
del yield_ims_2048
del yield_ims_3072
del yield_ims_4096
del dataloader_ims_seg_1728
del dataloader_ims_seg_2048
del dataloader_ims_seg_3072
del dataloader_ims_seg_4096
del dataloaders_all_sizes
libc = ctypes.CDLL("libc.so.6")
 libc.malloc_trim(0)
 gc.collect()
torch.cuda.empty_cache()
torch.cuda.empty_cache()<set_options>
train_datagen=ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range=0.2,
fill_mode='nearest'
)
libc = ctypes.CDLL("libc.so.6")
 libc.malloc_trim(0)
 gc.collect()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()<prepare_x_and_y>
ensem=10
model=[0]*ensem
for i in range(ensem):
model[i]=Sequential()
model[i].add(Conv2D(filters=32,kernel_size=(3,3),padding='same',activation='relu',input_shape=(28,28,1)))
 model[i].add(BatchNormalization())
 model[i].add(Conv2D(filters=32,kernel_size=(3,3),padding='same',activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(filters=32,kernel_size=(3,3),padding='same',activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(MaxPool2D(2,2))
model[i].add(Dropout(0.2))
model[i].add(Conv2D(filters=64,kernel_size=(3,3),padding='same',activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(filters=64,kernel_size=(3,3),padding='same',activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(filters=64,kernel_size=(3,3),padding='same',activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(MaxPool2D(2,2))
model[i].add(Dropout(0.2))
model[i].add(GlobalAveragePooling2D())
 model[i].add(Dense(128,activation='relu'))
model[i].add(Dropout(0.2))
model[i].add(Dense(10,activation='softmax'))
model[i].compile(loss='categorical_crossentropy',metrics=['accuracy'],optimizer='adam' )
X_test = data_df["ID"]<train_model>
callback=tf.keras.callbacks.EarlyStopping(monitor='accuracy',min_delta=0,patience=5,mode='auto',restore_best_weights=True,verbose=0)
 lrs=tf.keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x, verbose=0)
 history=[0]*ensem
for i in range(ensem):
train_x,valid_x,train_y,valid_y=train_test_split(x,y,test_size=0.2)
 history[i]=model[i].fit_generator(train_datagen.flow(train_x,train_y,batch_size=128),
epochs=100,
steps_per_epoch=train_x.shape[0]//128,
verbose=0,
validation_data=(valid_x,valid_y),
validation_steps=valid_x.shape[0]//128,
callbacks=[callback,lrs])
 print('Model {}: Epochs=100, Train_Accuracy:{}, Val_Accuracy:{}'.format(i+1,max(history[i].history['accuracy']),max(history[i].history['val_accuracy'])) )
<create_dataframe><EOS> 
prediction=np.zeros(( test.shape[0],10))
for i in range(ensem):
prediction=prediction+model[i].predict(test)
 prediction=np.argmax(prediction,axis = 1)
 prediction=pd.Series(prediction,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name="ImageId"),prediction],axis=1)
 submission.to_csv("digit_recognizer.csv",index=False )