Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<create_dataframe>
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
predictions = []
sub_df = pd.DataFrame(columns=["ID"], data=predict_ids_1728+predict_ids_2048+predict_ids_3072+predict_ids_4096)
 for size_idx, submission_ids in enumerate([predict_ids_1728, predict_ids_2048, predict_ids_3072, predict_ids_4096]):
size = IMAGE_SIZES[size_idx]
if submission_ids==[]:
print(f" 
...SKIPPING SIZE {size} AS THERE ARE NO IMAGE IDS...
 ")
 continue
else:
print(f" 
...WORKING ON IMAGE IDS FOR SIZE {size}...
 ")
 for i in tqdm(range(0, len(submission_ids), BATCH_SIZE), total=int(np.ceil(len(submission_ids)/BATCH_SIZE))):
batch_rgby_images = [
load_image(ID, TEST_IMG_DIR, testing=True, only_public=ONLY_PUBLIC)\
for ID in submission_ids[i:(i+BATCH_SIZE)]
]
if ONLY_PUBLIC:
batch_cell_bboxes = pub_ss_df[pub_ss_df.ID.isin(submission_ids[i:(i+BATCH_SIZE)])].mask_bboxes.values
batch_rgb_images = batch_rgby_images
submission_rles = pub_ss_df[pub_ss_df.ID.isin(submission_ids[i:(i+BATCH_SIZE)])].mask_sub_rles.values
if IS_DEMO:
batch_masks = [
sum([rle_to_mask(mask, size, size)for mask in batch])\
for batch in pub_ss_df[pub_ss_df.ID.isin(submission_ids[i:(i+BATCH_SIZE)])].mask_rles.values
]
else:
cell_segmentations = segmentator.pred_cells([[rgby_image[j] for rgby_image in batch_rgby_images] for j in [0, 3, 2]])
 nuc_segmentations = segmentator.pred_nuclei([rgby_image[2] for rgby_image in batch_rgby_images])
 batch_masks = [label_cell(nuc_seg, cell_seg)[1].astype(np.uint8)for nuc_seg, cell_seg in zip(nuc_segmentations, cell_segmentations)]
batch_rgb_images = [rgby_image.transpose(1,2,0)[..., :-1] for rgby_image in batch_rgby_images]
batch_cell_bboxes = [get_contour_bbox_from_raw(mask)for mask in batch_masks]
submission_rles = [[binary_mask_to_ascii(mask, mask_val=cell_id)for cell_id in range(1, mask.max() +1)] for mask in batch_masks]
batch_cell_tiles = [[
cv2.resize(
 pad_to_square(
 rgb_image[bbox[1]:bbox[3], bbox[0]:bbox[2],...]),
TILE_SIZE, interpolation=cv2.INTER_CUBIC)for bbox in bboxes]
for bboxes, rgb_image in zip(batch_cell_bboxes, batch_rgb_images)
 ]
if DO_TTA:
tta_batch_cell_tiles = [tta(tf.cast(ct, dtype=tf.float32), repeats=TTA_REPEATS)for ct in batch_cell_tiles]
else:
batch_cell_tiles = [tf.cast(ct, dtype=tf.float32)for ct in batch_cell_tiles]
if DO_TTA:
tta_batch_o_preds = [[inference_model.predict(ct)for ct in bct] for bct in tta_batch_cell_tiles]
batch_o_preds = [tf.keras.layers.Average()(tta_o_preds ).numpy() for tta_o_preds in tta_batch_o_preds]
else:
batch_o_preds = [inference_model.predict(cell_tiles)for cell_tiles in batch_cell_tiles]
batch_confs = [[pred[np.where(pred>CONF_THRESH)] for pred in o_preds] for o_preds in batch_o_preds]
batch_preds = [[np.where(pred>CONF_THRESH)[0] for pred in o_preds] for o_preds in batch_o_preds]
for j, preds in enumerate(batch_preds):
for k in range(len(preds)) :
if preds[k].size==0:
batch_preds[j][k]=np.array([18,])
 batch_confs[j][k]=np.array([1-np.max(batch_o_preds[j][k]),])
 if IS_DEMO:
print(" 
...DEMO IMAGES...
 ")
 for rgb_images, masks, preds, confs in zip(batch_rgb_images, batch_masks, batch_preds, batch_confs):
plot_predictions(rgb_images, masks, preds, confs=confs, fill_alpha=0.2, lbl_as_str=True)
 submission_rles = [flatten_list_of_lists([[m,]*len(p)for m, p in zip(masks, preds)])for masks, preds in zip(submission_rles, batch_preds)]
batch_preds = [flatten_list_of_lists(preds, to_string=True)for preds in batch_preds]
batch_confs = [[f"{conf:.4f}" for cell_confs in confs for conf in cell_confs] for confs in batch_confs]
predictions.extend([" ".join(flatten_list_of_lists(zip(*[preds,confs,masks])))for preds, confs, masks in zip(batch_preds, batch_confs, submission_rles)])
 sub_df["PredictionString"] = predictions
print(" 
...TEST DATAFRAME...
 ")
 display(sub_df.head(3))<save_to_csv>
train_data = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data = pd.read_csv('.. /input/digit-recognizer/test.csv')
 train_data.head()
ss_df = ss_df.merge(sub_df, how="left", on="ID")
 ss_df["PredictionString"] = ss_df.apply(create_pred_col, axis=1)
 ss_df = ss_df.drop(columns=["PredictionString_x", "PredictionString_y"])
 display(ss_df)
 torch.cuda.empty_cache()<categorify>
train_labels = train_data['label']
train_data = train_data.drop('label', axis=1 )
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset<feature_engineering>
encoder = LabelBinarizer()
train_labels = encoder.fit_transform(train_labels )
HPA_MODELS = False
COMPETITION_NAME = "hpa-single-cell-image-classification"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16
IMSIZE =(224, 240, 260, 300, 380, 456, 528, 600, 720)
 load_dir = f"/kaggle/input/{COMPETITION_NAME}/"
sub_df = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv')
 sub_df = sub_df.drop(sub_df.columns[1:],axis=1)
 for i in range(19):
sub_df[f'{i}'] = pd.Series(np.zeros(sub_df.shape[0]))
if HPA_MODELS:
colours = ['blue', 'green', 'red', 'yellow']
sub_dfs = []
for colour in colours:
test_paths = load_dir + "/test/" + sub_df['ID'] + '_' + colour + '.png'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[-1], IMSIZE[-1]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder 
)
 with strategy.scope() :
model = tf.keras.models.load_model(
 f'.. /input/hpa-models/effb7_individual_model_{colour}.h5' 
)
 model.summary()
sub_df[label_cols] = model.predict(dtest, verbose=1)
 sub_dfs.append(sub_df)
 if colour == 'yellow':
sub_df = pd.concat(( sub_df.ID,(
 sub_dfs[0].iloc[:, 1:] + sub_dfs[1].iloc[:, 1:] + sub_dfs[2].iloc[:, 1:] + sub_dfs[3].iloc[:, 1:] 
)/4), axis=1)
 else:
test_paths = load_dir + "/test/" + sub_df['ID'] + '_' + 'green' + '.png'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[-1], IMSIZE[-1]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder 
)
 with strategy.scope() :
model = tf.keras.models.load_model('.. /input/hpa-models/HPA classification efnb7 train 13cc0d 20/model_green.h5')
 model.summary()
sub_df[label_cols] = model.predict(dtest, verbose=1)
 ss_df = pd.merge(ss_df, sub_df, on = 'ID', how = 'left')
 for row in range(ss_df.shape[0]):
pred = ss_df.loc[row,'PredictionString']
pred_split = pred.split()
for j in range(int(len(pred_split)/3)) :
for k in range(19):
if int(pred_split[ 3*j ])== k:
p = pred_split[ 3*j + 1 ]
pred_split[ 3*j + 1 ] = str(ss_df.loc[row, f'{k}']*0.6 + float(p)*0.4)
 ss_df.loc[row,'PredictionString'] = ' '.join(pred_split)
 ss_df = ss_df[['ID','ImageWidth','ImageHeight','PredictionString']]
ss_df.to_csv('submission.csv',index = False)
 print(f'HPA_MODELS = {HPA_MODELS}' )<install_modules>
train_data = train_data.astype('float32')/ 255
test_data = test_data.astype('float32')/ 255
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<install_modules>
train_data = train_data[:, :, :, np.newaxis]
test_data = test_data[:, :, :, np.newaxis]
<define_variables>
X_train, X_val, y_train, y_val = train_test_split(train_data, train_labels, test_size=0.1, random_state=157, stratify=train_labels )
package_path = '.. /input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'
sys.path.append(package_path)
<import_modules>
data_generator = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
horizontal_flip=False,
vertical_flip=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1 
)
 data_generator.fit(X_train )
import pandas as pd
import numpy as np
from fastai.vision.all import *
import pickle
import os<import_modules>
init_relu = he_uniform(seed=157)
 init_tanh = glorot_uniform(seed=157)
 model = Sequential()
model.add(Conv2D(name='Conv_1', input_shape=(28, 28, 1), filters=64, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Conv2D(name='Conv_2', filters=64, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(MaxPooling2D(name='Pooling_1', pool_size=(2, 2), strides=2))
model.add(Dropout(0.20))
model.add(Conv2D(name='Conv_4', filters=32, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Conv2D(name='Conv_5', filters=32, kernel_size=(3, 3), padding='same', kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(MaxPooling2D(name='Pooling_2', pool_size=(2, 2), strides=2))
model.add(Dropout(0.20))
model.add(Flatten())
 model.add(Dense(name='FC_1', units=512, kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Dropout(0.20))
model.add(Dense(name='FC_2', units=128, kernel_initializer=init_relu, kernel_constraint=maxnorm(3)))
 model.add(BatchNormalization())
 model.add(Activation(relu))
model.add(Dense(name='output', units=10, activation='softmax', kernel_initializer=init_tanh, kernel_constraint=maxnorm(3)) )
<define_variables>
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ""
os.environ['PYTHONHASHSEED'] = str(157)
 random.seed(157)
 np.random.seed(157)
 tf.compat.v1.set_random_seed(157)
 session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 k.set_session(sess )
path = Path('.. /input/hpa-cell-tiles-sample-balanced-dataset' )<load_from_csv>
lr = 0.001
opt = Adam(learning_rate=lr, amsgrad=True)
 model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'])
 checkpoint = ModelCheckpoint('neural_network_checkpoint_training.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
 lr_decay = LearningRateScheduler(lambda x: lr * 0.95 ** x)
 tensorboard = TensorBoard(log_dir='./logs',
histogram_freq=0,
write_graph=True,
write_images=True)
 model.fit_generator(data_generator.flow(X_train, y_train, batch_size=64),
epochs=50,
validation_data=(X_val, y_val),
shuffle=True,
callbacks=[tensorboard, checkpoint, lr_decay],
verbose=1)
 save_model(checkpoint.model, 'neural_network_latest_saved.h5' )
df = pd.read_csv(path/'cell_df.csv' )<feature_engineering>
clf = load_model('neural_network_checkpoint_training.h5')
 prediction = clf.predict(test_data)
 prediction = pd.DataFrame(np.argmax(prediction, axis=-1), columns=['Label'])
 img_idx = pd.DataFrame(np.arange(1, len(prediction)+ 1), columns=['ImageId'])
 prediction = pd.concat([img_idx, prediction], axis=1)
 prediction.to_csv('submission_cnn.csv', index=False )