Kaggle competition: Digit Recognizer
os.environ['SM_FRAMEWORK'] = 'tf.keras' 
<load_pretrained>
pip install livelossplot
MODEL_PATH = '.. /input/train-fpn-segmentation-model-no-43/'
with open(MODEL_PATH+'hparams.json')as json_file:
hparams = json.load(json_file)
 hparams<normalization>
%matplotlib inline
IMG_SIZE = hparams['IMG_SIZE']
SCALE_FACTOR = hparams['SCALE_FACTOR']
K_SPLITS = hparams['K_SPLITS']
def read_tif_file(fname):
img = io.imread(fname)
 img = np.squeeze(img)
 if img.shape[0] == 3:
img = img.swapaxes(0,1)
 img = img.swapaxes(1,2)
 return img
def map_img2file(fname):
img = read_tif_file(fname)
 dims = np.array(img.shape)
 ch = 1 if len(dims)== 2 else dims[2]
for i in range(ch):
f = np.memmap('img{}.dat'.format(i), dtype=np.uint8, mode='w+', shape=(dims[0], dims[1]))
f[:] = img[:,:,i] if ch > 1 else img[:,:]
del f
return dims
def get_patch_from_file(dims, pos, psize):
ch = 1 if len(dims)== 2 else dims[2]
patch = np.zeros([psize[0], psize[1]], dtype=np.uint8)if ch == 1 else np.zeros([psize[0], psize[1], ch], dtype=np.uint8)
 for i in range(ch):
f = np.memmap('img{}.dat'.format(i), dtype=np.uint8, mode='r', shape=(dims[0], dims[1]))
p = f[pos[0]:pos[0]+psize[0], pos[1]:pos[1]+psize[1]]
crop = p.shape
if ch == 1:
patch[0:p.shape[0], 0:p.shape[1]] = p
else:
patch[0:p.shape[0], 0:p.shape[1],i] = p
del f
return patch, crop<categorify>
train_set = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_set = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 img_col = 28
img_row = 28
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<feature_engineering>
X_train_df = train_set.drop(['label'],axis = 1)
 y_train_df = train_set['label']
X_test_df = test_set
X_tr = np.asarray(X_train_df)/255
y_tr = np.asarray(y_train_df)
 X_te = np.asarray(X_test_df)/255
print(type(X_tr))
print(X_tr.shape)
 print(X_te.shape)
def create_TTA_batch(img):
if len(img.shape)< 4:
img = np.expand_dims(img, 0)
 batch=np.zeros(( img.shape[0]*8,img.shape[1],img.shape[2],img.shape[3]), dtype=np.float32)
 for i in range(img.shape[0]):
orig = tf.keras.preprocessing.image.img_to_array(img[i,:,:,:])/255.
 batch[i*8,:,:,:] = orig
batch[i*8+1,:,:,:] = np.rot90(orig, axes=(0, 1), k=1)
 batch[i*8+2,:,:,:] = np.rot90(orig, axes=(0, 1), k=2)
 batch[i*8+3,:,:,:] = np.rot90(orig, axes=(0, 1), k=3)
 orig = orig[:, ::-1]
batch[i*8+4,:,:,:] = orig
batch[i*8+5,:,:,:] = np.rot90(orig, axes=(0, 1), k=1)
 batch[i*8+6,:,:,:] = np.rot90(orig, axes=(0, 1), k=2)
 batch[i*8+7,:,:,:] = np.rot90(orig, axes=(0, 1), k=3)
 return batch
def mask_TTA(masks):
batch=np.zeros(( masks.shape[0],masks.shape[1],masks.shape[2],masks.shape[3]), dtype=np.float32)
 for i in range(masks.shape[0]//8):
batch[i*8,:,:,:] = masks[i*8]
batch[i*8+1,:,:,:] = np.rot90(masks[i*8+1], axes=(0, 1), k=3)
 batch[i*8+2,:,:,:] = np.rot90(masks[i*8+2], axes=(0, 1), k=2)
 batch[i*8+3,:,:,:] = np.rot90(masks[i*8+3], axes=(0, 1), k=1)
 batch[i*8+4,:,:,:] = masks[i*8+4][:, ::-1]
batch[i*8+5,:,:,:] = np.rot90(masks[i*8+5], axes=(0, 1), k=3)[:, ::-1]
batch[i*8+6,:,:,:] = np.rot90(masks[i*8+6], axes=(0, 1), k=2)[:, ::-1]
batch[i*8+7,:,:,:] = np.rot90(masks[i*8+7], axes=(0, 1), k=1)[:, ::-1]
return(batch )<save_to_csv>
train_set['label'].value_counts().sort_index()
MEANING_OF_LIFE = 42
MEANING_OF_LIFE_REV = int(str(MEANING_OF_LIFE)[::-1])
 PATH = '.. /input/hubmap-kidney-segmentation/test/'
filelist = glob.glob(PATH+'*.tiff')
 if len(filelist)== 5:
filelist = filelist[:1]
SUB_FILE = './submission.csv'
with open(SUB_FILE, 'w')as f:
f.write("id,predicted
")
 MODELS = [MODEL_PATH+'FPN-model-2']
s_th = MEANING_OF_LIFE+K_SPLITS
p_th = IMG_SIZE*IMG_SIZE//32
size = int(IMG_SIZE * SCALE_FACTOR)
 OVERLAP = size//2
STEP = size-OVERLAP
for file in filelist:
fid = file.replace('\','.' ).replace('/','.' ).split('.')[-2]
print(fid)
 dims = map_img2file(file)
 pmask = np.zeros(dims[:2], dtype=np.uint8)
 x_shft, y_shft = 0,0
if fid == 'afa5e8098':
x_shft, y_shft = MEANING_OF_LIFE, MEANING_OF_LIFE_REV
for modl in range(len(MODELS)) :
print(MODELS[modl])
 mname = MODELS[modl]
with open(mname+'.json', 'r')as m:
lm = m.read()
model = model_from_json(lm)
 model.load_weights(mname+'.h5')
 for x in range(( dims[0]-OVERLAP-x_shft)//STEP + min(1,(dims[0]-OVERLAP-x_shft)% STEP)) :
for y in range(( dims[1]-OVERLAP-y_shft)//STEP + min(1,(dims[1]-OVERLAP-y_shft)% STEP)) :
tile, crop = get_patch_from_file(dims, [x*STEP+x_shft, y*STEP+y_shft], [size,size])
 patch = cv2.resize(tile,
dsize=(IMG_SIZE, IMG_SIZE),
interpolation = cv2.INTER_AREA)
 _, s, _ = cv2.split(cv2.cvtColor(patch, cv2.COLOR_BGR2HSV))
if(s>s_th ).sum() > p_th:
batch = create_TTA_batch(patch)
 preds = model.predict(batch)
 pred = mask_TTA(preds)
 mask = np.rint(np.sum(pred, axis=0))
pint =cv2.resize(mask.astype(int), dsize=(size, size), interpolation = cv2.INTER_NEAREST)
 pmask[x*STEP:x*STEP+crop[0], y*STEP:y*STEP+crop[1]] += pint[0:crop[0], 0:crop[1]].astype(np.uint8)
 pmask = pmask >= MEANING_OF_LIFE_REV - K_SPLITS
with open(SUB_FILE, 'a')as f:
f.write("{},".format(fid))
f.write(rle_encode_less_memory(pmask))
f.write("
" )<set_options>
X_train_f = X_tr.reshape(42000,img_col,img_row,1)
 X_test_f= X_te.reshape(28000,img_col,img_row,1)
 y_train_f = to_categorical(y_tr)
 y_train_f.shape[1]
%rm -f *.dat<import_modules>
model = Sequential()
model.add(Conv2D(64,(4,4),padding = 'valid', input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(2,2),padding = 'same'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(2,2),padding = 'same'))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(512))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.1))
model.add(Dense(256))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.1))
model.add(Dense(128))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dense(64))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dense(64))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dense(10 , activation = 'softmax'))
<save_to_csv>
init_lr = 1e-2
decay_steps = 150
alpha = 1e-5
beta = 1e-8
num_periods=4
lin_cos_dec1 = tf.keras.experimental.LinearCosineDecay(init_lr,
decay_steps,
num_periods=num_periods, alpha=alpha,
beta=beta, name='LinCosDec 1' )
submit_file = '.. /input/hubmaplocal/FrogUnetR34_ASPP_AttDecode_final_thr0.4.csv'
submission = pd.read_csv(submit_file, index_col='id')
 sample_sub = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv', index_col='id')
 pub_ids = submission.index.values
predictions = submission.values
sample_sub.loc[pub_ids] = predictions
sample_sub.to_csv('submission.csv')
 print('done!')
 print(sample_sub )<install_modules>
opt = Adam(lr=0.00005)
 model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
 model.summary()
!pip install.. /input/packages/pretrainedmodels-0.7.4-py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch/segmentation_models/timm-0.1.20-py3-none-any.whl
!pip install.. /input/packages/efficientnet_pytorch-0.6.3-py2.py3-none-any.whl
!pip install.. /input/segmentationmodelspytorch/segmentation_models/segmentation_models_pytorch-0.1.2-py3-none-any.whl
clear_output()<install_modules>
epochs = 150
batch_size = 64
X_train, X_val, y_train, y_val = train_test_split(X_train_f, y_train_f, test_size=0.3 , random_state = 5)
 image_gen = ImageDataGenerator(rotation_range = 25 ,shear_range = 0.25,zoom_range = [1.25,0.75],width_shift_range= 0.1,height_shift_range=0.1)
 image_gen2 = ImageDataGenerator()
train_batches = image_gen.flow(X_train,y_train,batch_size = batch_size)
 val_batches =image_gen2.flow(X_val,y_val,batch_size = batch_size )
!pip install git+https://github.com/qubvel/segmentation_models.pytorch<load_from_csv>
steps_per_epoch = train_batches.n//train_batches.batch_size
validation_steps = val_batches.n//val_batches.batch_size
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.75,patience=3, min_lr=0.00001, mode='auto',verbose=1)
 callbacks = [PlotLossesKerasTF() , reduce_lr]
sample_submission = pd.read_csv('.. /input/hubmap-kidney-segmentation/sample_submission.csv')
 sample_submission = sample_submission.set_index('id')
 seed = 1015
np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def rle_encode_less_memory(img):

 pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 test_files = sample_submission.index.tolist()<import_modules>
history=model.fit_generator(generator=train_batches, steps_per_epoch = steps_per_epoch, epochs=epochs,
validation_data=val_batches, validation_steps=validation_steps , callbacks=callbacks )
<choose_model_class><EOS> 
predictions = model.predict_classes(X_test_f, verbose=0)
 submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("mysub5.csv", index=False, header=True )