Kaggle competition: Histopathologic Cancer Detection
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.3,
min_lr=0.00001)
 early_stopping = EarlyStopping(
 min_delta=0.000001,
patience=20,
restore_best_weights=True,
)<define_search_space>
df = pd.read_csv('.. /input/train_labels.csv')
 print('Shape of DataFrame',df.shape)
 df.head()
skf = StratifiedKFold(n_splits=3,random_state=42,shuffle=True)
 sub = pd.DataFrame(data=None, index=(range(1,28001)) , columns=None, dtype=None, copy=False)
 for train_index, val_index in skf.split(X, y):
model = keras.Sequential([
keras.layers.Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',
activation ='relu', input_shape =(28,28,1)) ,
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'),
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'),
keras.layers.MaxPool2D(pool_size=(2,2)) ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Flatten() ,
keras.layers.Dense(28*28, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Dense(28*28, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.4),
keras.layers.Dense(28*28, activation='relu'),
keras.layers.Dense(10, activation='softmax')
 ])
 X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 history = model.fit(
 x=X_train, y=y_train, batch_size=100, epochs=250, verbose=1, callbacks=[early_stopping,learning_rate_reduction],
validation_data=(X_val,y_val), shuffle=True)
 results = model.predict(X_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 sub = pd.concat([sub, results],axis=1)
<save_to_csv>
TRAIN_DIR = '.. /input/train/'
sub["result"] = sub.mode(dropna=True,axis=1)[0]
result = pd.Series(sub["result"],name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),result],axis = 1)
 submission = submission.dropna().astype('int32')
 submission.to_csv("mnist_ansamble_of_cnn.csv",index=False )<define_variables>
df[df['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']
df[df['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']
df.head()
IMG_SIZE =(96, 96)
 IN_SHAPE =(*IMG_SIZE, 3)
 BATCH_SIZE = 64<load_from_csv>
y = df_train['label']
df_train, df_val = train_test_split(df_train, test_size = 0.1, random_state = 0, stratify = y )
df_data = pd.read_csv('.. /input/train_labels.csv')
 train_dir = ".. /input/train/"
test_dir = ".. /input/test" 
<split>
df.set_index('id', inplace=True)
 train_list = list(df_train['id'])
 val_list = list(df_val['id'])
 for image in train_list:
file_name = image + '.tif'
target = df.loc[image,'label']
if target == 0:
label = '0'
elif target == 1:
label = '1'
src = os.path.join('.. /input/train', file_name)
 dest = os.path.join(train_dir, label, file_name)
 shutil.copyfile(src, dest)
 for image in val_list:
file_name = image + '.tif'
target = df.loc[image,'label']
if target == 0:
label = '0'
elif target == 1:
label = '1'
src = os.path.join('.. /input/train', file_name)
 dest = os.path.join(val_dir, label, file_name)
 shutil.copyfile(src, dest )
train, valid = train_test_split(df_data,test_size=0.15)
 train_datagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean())/ x.std() if x.std() > 0 else x,
horizontal_flip=True, vertical_flip=True,
rotation_range=90, shear_range=0.05, zoom_range=0.1)
 test_datagen = ImageDataGenerator(preprocessing_function=lambda x:(x - x.mean())/ x.std() if x.std() > 0 else x)
 train_generator = train_datagen.flow_from_dataframe(
 dataframe = train,
directory='.. /input/train/',
x_col='id',
y_col='label',
has_ext=False,
batch_size=BATCH_SIZE,
seed=2018,
shuffle=True,
class_mode='binary',
target_size=IMG_SIZE)
 valid_generator = test_datagen.flow_from_dataframe(
 dataframe = valid,
directory='.. /input/train/',
x_col='id',
y_col='label',
has_ext=False,
batch_size=BATCH_SIZE,
seed=2018,
shuffle=False,
class_mode='binary',
target_size=IMG_SIZE
)<choose_model_class>
IMAGE_SIZE = 96
train_path = 'base_dir/train_dir'
valid_path = 'base_dir/val_dir'
test_path = '.. /input/test'
num_train_samples = len(df_train)
 num_val_samples = len(df_val)
 train_batch_size = 32
val_batch_size = 32
train_steps = np.ceil(num_train_samples / train_batch_size)
 val_steps = np.ceil(num_val_samples / val_batch_size)
 datagen = ImageDataGenerator(rescale=1.0/255)
 train_gen = datagen.flow_from_directory(train_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=train_batch_size,
class_mode='categorical')
 val_gen = datagen.flow_from_directory(valid_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=val_batch_size,
class_mode='categorical')
 test_gen = datagen.flow_from_directory(valid_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=1,
class_mode='categorical',
shuffle=False )
conv_base = VGG16(include_top = False,
input_shape = IN_SHAPE,
weights='imagenet')
 for layer in conv_base.layers[:-12]:
layer.trainable = False
for layer in conv_base.layers:
print(layer, layer.trainable )<choose_model_class>
import keras
from keras.models import Sequential
from keras.layers import Conv2D, Dropout, MaxPooling2D, Flatten, Dense
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import SeparableConv2D
from keras.layers.core import Activation
model = Sequential()
model.add(conv_base)
 model.add(Flatten())
 model.add(Dense(1024, activation="relu"))
model.add(Dropout(0.6))
model.add(BatchNormalization())
 model.add(Dense(512, activation="relu"))
model.add(Dropout(0.6))
model.add(BatchNormalization())
 model.add(Dense(1, activation = "sigmoid"))
conv_base.summary()<choose_model_class>
class Net:
@staticmethod
def build(width, height, depth, classes):
model = Sequential()
inputShape =(height, width, depth)
 model.add(Conv2D(filters = 32, kernel_size =(5,5), padding="same", activation='relu', input_shape= inputShape))
model.add(Conv2D(filters = 32, kernel_size =(3,3), padding="same", activation='relu'))
model.add(Conv2D(filters = 32, kernel_size =(3,3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding="same", activation='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding="same", activation='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(filters = 128, kernel_size =(3,3), padding="same", activation='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3), padding="same", activation='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(units = 500, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(classes, activation='softmax'))
model.summary()
return model
model.compile(optimizers.Adam(0.001), loss = "binary_crossentropy", metrics=["accuracy"] )<train_model>
class CancerNet:
@staticmethod
def build(width, height, depth, classes):
model = Sequential()
inputShape =(height, width, depth)
 chanDim = -1
model.add(SeparableConv2D(32,(3, 3), padding="same",input_shape = inputShape))
model.add(Activation("relu"))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(SeparableConv2D(64,(3, 3), padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(axis=chanDim))
model.add(SeparableConv2D(64,(3, 3), padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(SeparableConv2D(128,(3, 3), padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(axis=chanDim))
model.add(SeparableConv2D(128,(3, 3), padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(axis=chanDim))
model.add(SeparableConv2D(128,(3, 3), padding="same"))
model.add(Activation("relu"))
model.add(BatchNormalization(axis=chanDim))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256))
model.add(Activation("relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(classes))
model.add(Activation("softmax"))
model.summary()
return model
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
earlystopper = EarlyStopping(monitor='val_loss', patience=3, verbose=1, restore_best_weights=True)
 learning_rate_decay = ReduceLROnPlateau(monitor='acc', patience=2, verbose=1, factor=0.3, min_lr=1e-5)
 history = model.fit_generator(train_generator, steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=50,
callbacks=[learning_rate_decay, earlystopper] )<feature_engineering>
model = Net.build(width = 96, height = 96, depth = 3, classes = 2)
 model.compile(optimizer = Adam(lr=0.0001), loss = 'binary_crossentropy', metrics=['accuracy'] )
base_test_dir = '.. /input/test/'
test_files = glob(os.path.join(base_test_dir,'*.tif'))
submission = pd.DataFrame()
file_batch = 5000
max_idx = len(test_files)
 for idx in range(0, max_idx, file_batch):
print("Indexes: %i - %i"%(idx, idx+file_batch))
test_df = pd.DataFrame({'path': test_files[idx:idx+file_batch]})
 test_df['id'] = test_df.path.map(lambda x: x.split('/')[3].split(".")[0])
 test_df['image'] = test_df['path'].map(imread)
 K_test = np.stack(test_df["image"].values)
 K_test =(K_test - K_test.mean())/ K_test.std()
predictions = model.predict(K_test)
 test_df['label'] = predictions
submission = pd.concat([submission, test_df[["id", "label"]]])
 submission.head()<save_to_csv>
submission.to_csv("submission.csv", index = False, header = True )<define_variables>
filepath = "checkpoint.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose = 1,
save_best_only = True, mode = 'max')
 reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor = 0.5, patience = 2, verbose = 1, mode = 'max', min_lr = 0.00001)
 callbacks_list = [checkpoint, reduce_lr]
history = model.fit_generator(train_gen, steps_per_epoch = train_steps,
validation_data = val_gen,
validation_steps = val_steps,
epochs = 11,
verbose = 1,
callbacks = callbacks_list )
data = pd.read_csv('/kaggle/input/train_labels.csv')
 train_path = '/kaggle/input/train/'
test_path = '/kaggle/input/test/'
data['label'].value_counts()<split>
model.load_weights('checkpoint.h5')
 val_loss, val_acc = \
model.evaluate_generator(test_gen, steps=len(df_val))
print('val_loss:', val_loss)
 print('val_acc:', val_acc )
def readImage(path):
bgr_img = cv2.imread(path)
 b,g,r = cv2.split(bgr_img)
 rgb_img = cv2.merge([r,g,b])
 return rgb_img<split>
predictions = model.predict_generator(test_gen, steps=len(df_val), verbose=1 )
train_df = data.set_index('id')
 train_names = train_df.index.values
train_labels = np.asarray(train_df['label'].values)
 tr_n, tr_idx, val_n, val_idx = train_test_split(train_names, range(len(train_names)) , test_size=0.2, stratify=train_labels, random_state=123 )<define_variables>
df_preds = pd.DataFrame(predictions, columns=['no_tumor', 'has_tumor'])
 df_preds.head()
arch = squeezenet1_1
BATCH_SIZE = 128
sz = CROP_SIZE
MODEL_PATH = str(arch ).split() [1]<create_dataframe>
y_true = test_gen.classes
y_pred = df_preds['has_tumor']
train_dict = {'name': train_path + train_names, 'label': train_labels}
df = pd.DataFrame(data=train_dict)
 test_names = []
for f in os.listdir(test_path):
test_names.append(test_path + f)
 df_test = pd.DataFrame(np.asarray(test_names), columns=['name'] )<train_model>
print('ROC AUC Score = ',roc_auc_score(y_true, y_pred))
class MyImageItemList(ImageList):
def open(self, fn:PathOrStr)->Image:
img = readCroppedImage(fn.replace('/./','' ).replace('//','/'))
return vision.Image(px=pil2tensor(img, np.float32))<categorify>
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_true, y_pred)
 auc_keras = auc(fpr_keras, tpr_keras )
imgDataBunch =(MyImageItemList.from_df(path='/', df=df, suffix='.tif')
.split_by_idx(val_idx)
.label_from_df(cols='label')
.add_test(MyImageItemList.from_df(path='/', df=df_test)) 
.transform(tfms=[[],[]], size=sz)
.databunch(bs=BATCH_SIZE)
.normalize([tensor([0.702447, 0.546243, 0.696453]), tensor([0.238893, 0.282094, 0.216251])])
 )<define_variables>
report = classification_report(y_true, y_pred_binary, target_names = ['no_tumor', 'has_tumor'])
 print(report )
imgDataBunch.show_batch(rows=2, figsize=(4,4))<choose_model_class>
test_path ='test_dir'
test_gen = datagen.flow_from_directory(test_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=1,
class_mode='categorical',
shuffle=False )
def getLearner() :
return create_cnn(imgDataBunch, arch, pretrained=True, path='.', metrics=accuracy, ps=0.5, callback_fns=ShowGraph)
 learner = getLearner()<choose_model_class>
num_test_images = 57458
predictions = model.predict_generator(test_gen, steps=num_test_images, verbose=1 )
lrs = []
losses = []
wds = []
iter_count = 600
learner.lr_find(wd=1e-6, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-6')
 learner = getLearner()
learner.lr_find(wd=1e-4, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-4')
 learner = getLearner()
learner.lr_find(wd=1e-2, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-2')
 learner = getLearner()<train_model>
if predictions.shape[0] == num_test_images:
print('All Predictions Done!')
 else:
print('Error!' )
max_lr = 2e-2
wd = 1e-4
learner.fit_one_cycle(cyc_len=8, max_lr=max_lr, wd=wd )<save_model>
df_preds = pd.DataFrame(predictions, columns=['no_tumor', 'has_tumor'])
 df_preds.head()
learner.save(MODEL_PATH + '_stage1' )<train_model>
test_filenames = test_gen.filenames
df_preds['file_names'] = test_filenames
def extract_id(x):
a = x.split('/')
 b = a[1].split('.')
 extracted_id = b[0]
return extracted_id
df_preds['id'] = df_preds['file_names'].apply(extract_id)
 df_preds.head()
learner.fit_one_cycle(cyc_len=12, max_lr=slice(4e-5,4e-4))<save_model>
y_pred = df_preds['has_tumor']
image_id = df_preds['id']
learner.save(MODEL_PATH + '_stage2' )<compute_test_metric>
submission = pd.DataFrame({'id':image_id,
'label':y_pred,
} ).set_index('id')
 submission.to_csv('submission.csv', columns=['label'] )
<categorify><EOS> 
shutil.rmtree('test_dir' )