Kaggle competition: RSNA Intracranial Hemorrhage Detection
<SOS> metric: weightedmeancolumnwiselogloss Kaggle data source: rsna-intracranial-hemorrhage-detection<set_options>
!pip install git+https://github.com/fastai/fastai_dev
%matplotlib inline
np.random.seed(0 )<import_modules>
dir_csv = '.. /input/rsna-intracranial-hemorrhage-detection'
dir_train_img = '.. /input/rsna-train-stage-1-images-png-224x/stage_1_train_png_224x'
dir_test_img = '.. /input/rsna-test-stage-1-images-png-224x/stage_1_test_png_224x'
dir_train_csv = ".. /input/rsnsa-intracranial-hemorrahage-detection/stage_1_train.csv"
dir_sample_submission_csv = ".. /input/rsnsa-intracranial-hemorrahage-detection/stage_1_sample_submission.csv"
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools<import_modules>
from fastai2.torch_basics import *
from fastai2.test import *
from fastai2.layers import *
from fastai2.data.all import *
from fastai2.optimizer import *
from fastai2.learner import *
from fastai2.metrics import *
from fastai2.vision.all import *
from fastai2.vision.learner import *
from fastai2.vision.models import *
from fastai2.callback.all import *
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau<load_from_csv>
items = get_image_files(dir_train_img)
 items = [i for i in items if '(copy)' not in i.name]
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 X = train_data.drop(labels = ['label'], axis= 1)
 y = train_data['label']
del train_data
sns.countplot(y )<feature_engineering>
!mkdir -p data
df_train = pd.read_csv(f'{dir_csv}/stage_1_train.csv')
 df_train.head()
X = X/255.0
test_data = test_data/255.0<categorify>
df_train.Label.isna().sum()
y = to_categorical(y, num_classes=10 )<split>
df_train['fn'] = df_train.ID.apply(lambda x: '_'.join(x.split('_')[:2])+ '.png')
 df_train.columns = ['ID', 'probability', 'fn']
df_train['label'] = df_train.ID.apply(lambda x: x.split('_')[-1])
 df_train.drop_duplicates('ID', inplace=True )
random_seed =5
X_train,X_val,y_train, y_val =train_test_split(X,y,random_state=random_seed, test_size=0.08 )<choose_model_class>
pivot = df_train.pivot(index='fn', columns='label', values='probability')
 pivot.reset_index(inplace=True)
 pivot.to_csv('data/train_pivot.csv', index=False)
 pivot.head()
model = Sequential()
model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
ks, vs = [], []
for k, v in d.items() :
ks.append(k), vs.append(' '.join(v))
pd.DataFrame(data = {'fn': ks, "labels": vs} ).to_csv('data/train_labels_as_string.csv', index = False )
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
class Labeller() :

 def __init__(self):
self.df = pd.read_csv("data/train_labels_as_string.csv")
 self.df.set_index("fn", inplace = True)
 def __call__(self, path):
fn = path.name
labels_txt = self.df.loc[fn].labels
if isinstance(labels_txt, float)or labels_txt == ' ': return []
return labels_txt.split(' ' )
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
classes = L(pd.read_csv("data/train_pivot.csv" ).columns.tolist() [1:])
 classes
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
mcat = MultiCategorize(vocab = classes)
 mcat.o2i = classes.val2idx()
epochs = 35
batch_size = 64<init_hyperparams>
mcat.o2i
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1, 
)
 datagen.fit(X_train )<train_model>
tfms = [PILImage.create, [Labeller() , mcat, OneHotEncode() ]]
ds_img_tfms = [ToTensor() ]
dsrc = DataSource(items, tfms, splits = RandomSplitter()(items))
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
test_paths = get_image_files(dir_test_img)
 test_tfms = [PILImage.create, [lambda x: np.array([0,0,0,0,0,0])]]
dsrc_test = DataSource([test_paths[0]] + test_paths, test_tfms, splits=[[0], L(range(len(test_paths)) ).map(lambda x: x + 1)] )
results = model.predict(test_data)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
ds_img_tfms = [ToTensor() ]
dl_tfms = [Cuda() , IntToFloatTensor() , Normalize(*dataset_stats)]
dbch = dsrc.databunch(after_item = ds_img_tfms, after_batch = dl_tfms, bs = 128, num_workers = 4)
 dbch_test = dsrc_test.databunch(after_item = ds_img_tfms, after_batch = dl_tfms, bs = 128, num_workers = 4 )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("digit_cnn_keras.csv",index=False )<set_options>
model = create_cnn_model(resnet18, 6, -2, True )
warnings.simplefilter(action='ignore', category=FutureWarning)
 %matplotlib inline
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
np.set_printoptions(suppress=True, linewidth=np.nan, threshold=np.nan)
 train_file_csv = ".. /input/train.csv"
test_file_csv = ".. /input/test.csv"
x_train_file_npy = "x_train.npy"
y_train_file_npy = "y_train.npy"
x_test_file_npy = "x_test.npy"
x_train_cleaned_file_npy = "x_train_cleaned.npy"
y_train_cleaned_file_npy = "y_train_cleaned.npy"
model_path = ""
log_file = "log.txt"
log_mode = 'both'
def log_msg(msg=''):
fmt = "%H:%M:%S"
s = f'{datetime.today().strftime(fmt)}: {msg}'
if log_mode == 'file_only' or log_mode == 'both':
with open(log_file, "a")as myfile:
myfile.write(f'{s}
')
 if log_mode == 'screen_only' or log_mode == 'both':
print(s)
 log_msg('Script started.' )<load_from_csv>
model_segments = model[0][:6], model[0][6:], model[1]
if not os.path.isfile(x_train_file_npy):
mnist_train_dataset = read_csv(train_file_csv, delimiter=',' ).values
y_train = mnist_train_dataset[:, 0]
y_train = y_train.astype('uint8')
 x_train = mnist_train_dataset[0:, 1:]
x_train = x_train.astype('uint8')
 np.save(x_train_file_npy, x_train)
 np.save(y_train_file_npy, y_train)
 if not os.path.isfile(x_test_file_npy):
mnist_test_dataset = read_csv(test_file_csv, delimiter=',' ).values
x_test = mnist_test_dataset
x_test = x_test.astype('uint8')
 np.save(x_test_file_npy, x_test)
<train_model>
def trainable_params_mod(model): return L(trainable_params(segment)for segment in model_segments)
 opt_func = partial(Adam, wd=0.01, eps=1e-3 )
x_train = np.load(file=x_train_file_npy)
 y_train = np.load(file=y_train_file_npy)
 log_msg(f'x_train: {x_train.shape}.These are the training images.')
 log_msg(f'y_train: {y_train.shape}.These are the training labels.' )<define_variables>
learn = Learner(
 dbch,
model,
loss_func=BCEWithLogitsLossFlat() ,
metrics=[accuracy_multi],
opt_func=opt_func,
splitter=trainable_params_mod
)
lst = [12817,
60, 191, 2284, 2316, 5275, 7389, 19633, 19979, 24891, 29296, 32565, 38191, 38544, 40339, 41739,
4677, 7527, 9162, 13471, 16598, 20891, 27364,
240, 11593, 11896, 17966, 25708, 28560, 33198, 34477, 36018, 41492,
1383, 6781, 22478, 23604, 26171, 26182, 26411, 18593, 34862, 36051, 36241, 36830, 37544,
456, 2867, 2872, 5695, 6697, 9195, 18319, 19364, 27034, 29253, 35620,
7610, 12388, 12560, 14659, 15219, 18283, 24122, 31649, 40214, 40358, 40653,
6295, 7396, 15284, 19880, 20089, 21423, 25233, 26366, 26932, 27422, 31741,
8566, 10920, 23489, 25069, 28003, 28851, 30352, 30362, 35396, 36984, 39990, 40675, 40868, 41229,
631, 4226, 9943, 14914, 15065, 17300, 18316, 19399, 20003, 20018, 23135, 23732, 29524, 33641, 40881, 41354
]
x_cleaned = np.delete(x_train, lst, 0)
 y_cleaned = np.delete(y_train, lst, 0)
 np.save(x_train_cleaned_file_npy, x_cleaned)
 np.save(y_train_cleaned_file_npy, y_cleaned )<prepare_x_and_y>
learn.freeze_to(-1 )
x_train = np.load(file=x_train_cleaned_file_npy)
 y_train = np.load(file=y_train_cleaned_file_npy)
 log_msg(f'There are {x_train.shape[0]} training images.' )<split>
learn.lr_find()
random_seed = np.random.seed(2)
 x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.1, random_state=random_seed,
stratify=y_train )<data_type_conversions>
learn.fit_one_cycle(2, 2e-2 )
log_msg('')
 log_msg('Changing data type to float and normalizing values...')
 x_train = x_train.astype('float32')
 x_train /= 255
x_val = x_val.astype('float32')
 x_val /= 255
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
 x_val = x_val.reshape(x_val.shape[0], 28, 28, 1)
 log_msg(f'x_train shape is {x_train.shape} of data type {x_train.dtype}.These are our TRAINING images.')
 log_msg(f'x_val shape is {x_val.shape} of data type {x_val.dtype}.These are our VALIDATION images.')
 y_train = np_utils.to_categorical(y_train, 10)
 y_train = y_train.astype('float32')
 y_val = np_utils.to_categorical(y_val, 10)
 y_val = y_val.astype('float32')
 log_msg(f'y_train shape is {y_train.shape} of data type {y_train.dtype}.These are our TRAINING labels.')
 log_msg(f'y_val shape is {y_val.shape} of data type {y_val.dtype}.These are our VALIDATION labels.' )<train_model>
learn.save('phase-1' )
do_training = True
if do_training:
log_msg('---- TRAINING BEGIN ----')
 format_metric_val = '{:1.10f}'
metric = 'val_acc'
if metric == 'val_acc':
best_metric_val_so_far = 0
snifftest_max_epoch = 0
snifftest_metric_val = 0
elif metric == 'val_loss':
best_metric_val_so_far = 100.0
snifftest_max_epoch = 0
snifftest_metric_val = 100.0
iter_best_fit = 0
max_duration_mins = 330
fmt = "%a %b %d %H:%M:%S"
if max_duration_mins == 0:
started_at = 0
finish_by = 0
else:
started_at = datetime.today()
finish_by = started_at + timedelta(minutes=max_duration_mins)
 log_msg(f'Started at {started_at.strftime(fmt)}, finish by {finish_by.strftime(fmt)}')
 x = 3
epochs=500
patience=30
batch_size=500
for xtr in range(0, x):
K.clear_session()
log_msg(f'Iteration {xtr} of {x - 1}')
 input = Input(shape=(28, 28, 1))
x1 = Conv2D(32,(5, 5), padding='same', kernel_initializer='he_normal' )(input)
 x1 = Activation('relu' )(x1)
 x1 = Conv2D(64,(3, 3), padding='same', kernel_initializer='he_normal' )(x1)
 x1 = Activation('relu' )(x1)
 x1 = MaxPooling2D(pool_size=(2, 2))(x1)
 x1 = Dropout(0.5 )(x1)
 x1 = Conv2D(64,(3, 3), padding='same', kernel_initializer='he_normal' )(x1)
 x1 = Activation('relu' )(x1)
 x1 = Conv2D(32,(5, 5), padding='same', kernel_initializer='he_normal' )(x1)
 x1 = Activation('relu' )(x1)
 x1 = Dropout(0.5 )(x1)
 x1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x1)
 x1 = Dropout(0.5 )(x1)
 x1 = Flatten()(x1)
 x1 = Dense(128, activation='relu', kernel_initializer='he_normal' )(x1)
 x1 = Dropout(0.5 )(x1)
 output = Dense(10, activation='softmax', kernel_initializer='he_normal' )(x1)
 model = Model(inputs=[input], outputs=[output])
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 results, log = kbf.find_best_fit(model=model, metric=metric, xtrain=x_train, ytrain=y_train, xval=x_val,
yval=y_val, validation_split=0, batch_size=batch_size, epochs=epochs,
patience=patience, snifftest_max_epoch=snifftest_max_epoch,
snifftest_metric_val=snifftest_metric_val,
show_progress=True, format_metric_val=format_metric_val,
save_best=True, save_path=model_path,
best_metric_val_so_far=best_metric_val_so_far,
logmsg_callback=log_msg, finish_by=finish_by)
 del model
is_best = False
if metric == 'val_acc':
is_best = results['best_metric_val'] > best_metric_val_so_far
elif metric == 'val_loss':
is_best = results['best_metric_val'] < best_metric_val_so_far
if is_best:
iter_best_fit = xtr
best_metric_val_so_far = results['best_metric_val']
sbest_metric_val_so_far = metric + '=' + format_metric_val.format(best_metric_val_so_far)
 sbest_epoch = results['best_epoch']
best_log=results['history']
best_epoch=results['best_epoch']
s = f'NEW BEST SO FAR: {sbest_metric_val_so_far} on epoch {sbest_epoch}
'
log_msg(s)
 if results['expired']:
break
log_msg(f'The best result is {sbest_metric_val_so_far}')
 log_msg('---- TRAINING END ----' )<load_pretrained>
learn.load('phase-1');
x_test = np.load(file=x_test_file_npy)
 x_test = x_test.astype('float32')
 x_test /= 255
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
 y_test = None
with open('.//model.json', 'r')as f:
modelx = model_from_json(f.read())
 modelx.load_weights('.//model.hdf5')
<prepare_x_and_y>
learn.freeze_to(-2 )
<predict_on_test>
learn.lr_find(start_lr=1e-8, end_lr=1e-1 )
log_msg('Predicting a single test image...')
 img = x_test[0]
plt.imshow(img.reshape([28, 28]), cmap=plt.get_cmap('gray_r'))
plt.show()
img = img.reshape(1, 28, 28, 1)
 prediction = modelx.predict([img])
 predicted_classes = np.argmax(prediction[0], axis=0)
 log_msg('Probabilities')
 for x in range(0,10):
s = f'Class {x}: {format_metric_val.format(prediction[0,x])}'
log_msg(s)
 log_msg(f'Predicted class: {predicted_classes}')
<save_to_csv>
learn.fit_one_cycle(2, [1e-3, 1e-4, 1e-5] )
log_msg('Predicting all test images...')
 test_labels = modelx.predict(x_test)
 predicted_classes = np.argmax(test_labels, axis=1)
 log_msg('Prediction complete.' )<save_to_csv>
learn.save('phase-2')
 learn.load('phase-2');
submission = pd.DataFrame({
'ImageID': range(1, 28001),
'Label': predicted_classes
})
 submission.to_csv('submission.csv', index=False)
 log_msg('Saved predictions as submission.csv')
 log_msg('Script ended.' )<set_options>
learn.fit_one_cycle(5, [1e-4, 5e-4, 1e-3] )
np.set_printoptions(suppress=True)
<load_from_csv>
learn.save('phase-3' )
df1 = pd.read_csv('.. /input/train.csv')
 df2 = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
learn.load('phase-3' )
train_data = np.array(df1)
 test_data = np.array(df2)
 X_train_orig = np.asarray(train_data[:, 1::], dtype=np.float32)
 Y_train_orig = np.asarray(train_data[:, 0], dtype=np.float32)
 X_test_orig = np.asarray(test_data[:, 0::], dtype=np.float32 )<feature_engineering>
learn.dbunch = dbch_test
X_train = X_train_orig / 255.
 X_test = X_test_orig / 255 .<categorify>
preds, targs = learn.get_preds()
X_train = X_train.reshape(-1,28,28,1)
 X_test = X_test.reshape(-1,28,28,1)
 Y_train = Y_train_orig.reshape(-1, 1)
 Y_train = to_categorical(Y_train, num_classes=10 )<split>
ids = []
labels = []
for path, pred in zip(test_paths, preds):
for i, label in enumerate(classes):
ids.append(f"{path.name.split('.')[0]}_{label}")
 predicted_probability = '{0:1.10f}'.format(pred[i].item())
 labels.append(predicted_probability )
<train_model><EOS> 
pd.DataFrame({'ID': ids, 'Label': labels} ).to_csv(f'submission.csv', index=False )