Kaggle competition: Kannada MNIST
def do_fit(bs,sz,epochs,lr, freeze=True):
learn.dbunch = get_data(bs, sz)
 if freeze:
if learn.opt is not None: learn.opt.clear_state()
learn.freeze()
learn.fit_one_cycle(1, slice(lr))
learn.unfreeze()
learn.fit_one_cycle(epochs, slice(lr))<string_transform>
import random
import keras
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import matplotlib.patches as mpatches
from skimage.filters import threshold_otsu
from skimage.segmentation import clear_border
from skimage.measure import label, regionprops
from skimage.morphology import closing, square
from skimage.color import label2rgb
from math import sqrt
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
import itertools
from keras.utils.np_utils import to_categorical
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, Input, BatchNormalization
from keras.optimizers import RMSprop, Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import LearningRateScheduler
from keras.callbacks import ModelCheckpoint
fns = L(list(df_comb.fname)).map(filename)
 splits = split_data(df_comb )<feature_engineering>
keras.backend.clear_session()
def fix_pxrepr(dcm):
if dcm.PixelRepresentation != 0 or dcm.RescaleIntercept<-100: return
x = dcm.pixel_array + 1000
px_mode = 4096
x[x>=px_mode] = x[x>=px_mode] - px_mode
dcm.PixelData = x.tobytes()
dcm.RescaleIntercept = -1000<normalization>
train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')
 y_train = train['label']
X = train.drop(['label'],axis=1)
 del train
Id = test['id']
test = test.drop(['id'],axis=1 )
def dcm_tfm(fn):
fn =(path_trn/fn ).with_suffix('.dcm')
 try:
x = fn.dcmread()
fix_pxrepr(x)
 except Exception as e:
print(fn,e)
 raise SkipItemException
if x.Rows != 512 or x.Columns != 512: x.zoom_to(( 512,512))
px = x.scaled_px
return TensorImage(px.to_3chan(dicom_windows.brain,dicom_windows.subdural, bins=bins))<define_variables>
first_1000 = y_train[:10]
np.unique(first_1000, return_counts=True )
tfms = [[dcm_tfm], [fn2label,EncodedMultiCategorize(htypes)]]
dsrc = DataSource(fns, tfms, splits=splits)
 batch_tfms = [nrm, Cuda() , *aug]<prepare_x_and_y>
labels =[]
frequencies = []
for i in range(len(y_train)) :
lab, freq = str(y_train[i]), len([n for n in X.values[i] if n > 0])
 labels.append(lab)
 frequencies.append(freq)
 data = {'Labels':labels, 'Frequencies':frequencies}
df = pd.DataFrame(data )
def get_data(bs, sz):
return dsrc.databunch(bs=bs, num_workers=nw, after_batch=batch_tfms+[AffineCoordTfm(size=sz)] )<prepare_x_and_y>
df.groupby('Labels' ).mean()
dbch = get_data(64,256)
 x,y = to_cpu(dbch.one_batch())
 dbch.show_batch(max_n=4)
 x.shape<find_best_params>
test_y = df['Labels']
test_x = df
test_x.drop('Labels', axis =1, inplace = True)
learn.loss_func = get_loss(1.0 )<train_model>
X_train1, X_val1, y_train1, y_val1 = train_test_split(test_x, test_y, test_size=0.1, random_state=1337 )
def fit_tune(bs, sz, epochs, lr):
dbch = get_data(bs, sz)
 learn.dbunch = dbch
learn.opt.clear_state()
learn.unfreeze()
learn.fit_one_cycle(epochs, slice(lr))<define_variables>
clf_rfc = RandomForestClassifier(n_estimators = 100)
 clf_rfc.fit(X_train1,y_train1)
 accuracy_score(clf_rfc.predict(X_val1), y_val1)
test_fns = [(path_tst/f'{filename(o)}.dcm' ).absolute() for o in df_tst.fname.values]<compute_test_metric>
X = X.values.reshape(X.shape[0], 28, 28,1)
 test = test.values.reshape(test.shape[0], 28, 28,1)
 X = X / 255.0
test = test / 255.0
tst = test_dl(dbch, test_fns)
 x = tst.one_batch() [0]
x.min() ,x.max()<predict_on_test>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.25,
width_shift_range=0.25,
height_shift_range=0.25,
horizontal_flip=False,
vertical_flip=False)
 valid_datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.25,
width_shift_range=0.25,
height_shift_range=0.25,
horizontal_flip=False,
vertical_flip=False )
preds,targs = learn.get_preds(dl=tst)
 preds_clipped = preds.clamp (.0001,.999 )<define_variables>
y_train = to_categorical(y_train,num_classes=10 )
ids = []
labels = []
for idx,pred in zip(df_tst.index, preds_clipped):
for i,label in enumerate(htypes):
ids.append(f"{idx}_{label}")
 predicted_probability = '{0:1.10f}'.format(pred[i].item())
 labels.append(predicted_probability )<import_modules>
learning_rate_reduction = ReduceLROnPlateau(monitor='accuracy',
patience=3,
verbose=10,
factor=0.33,
min_lr=0.00001)
 checkpoint=ModelCheckpoint('bestweights.md5', monitor='accuracy', verbose=1, save_best_only=True)
from IPython.display import FileLink, FileLinks 
<define_variables>
epochs = 40
initial_learningrate=2e-3
batch_size = 100
dir_csv = '.. /input/rsna-intracranial-hemorrhage-detection'
dir_train_img = '.. /input/rsna-train-stage-1-images-png-224x/stage_1_train_png_224x'
dir_test_img = '.. /input/rsna-test-stage-1-images-png-224x/stage_1_test_png_224x' 
<define_variables>
X_train, X_val, y_train, y_val = train_test_split(X, y_train, test_size=0.1, random_state=1 )
n_classes = 6
n_epochs = 2
batch_size = 64 
<install_modules>
def build_model1(input_shape=(28, 28, 1), classes = 10):
activation = 'relu'
padding = 'same'
gamma_initializer = 'uniform'
input_layer = Input(shape=input_shape)
 hidden=Conv2D(32,(11,11), strides=1, padding =padding, activation = activation, name = 'conv1' )(input_layer)
 hidden=BatchNormalization()(hidden)
 hidden=Conv2D(32,(11,11), strides=1, padding =padding, activation = activation, name = 'conv2' )(hidden)
 hidden=BatchNormalization()(hidden)
 hidden=Conv2D(32,(5,5), strides=1, padding =padding, activation = activation, name = 'conv3' )(hidden)
 hidden=BatchNormalization()(hidden)
 hidden=MaxPool2D()(hidden)
 hidden=Dropout(0.35 )(hidden)
 hidden=Conv2D(64,(3,3), strides=1, padding =padding, activation = activation, name = 'conv4' )(input_layer)
 hidden=BatchNormalization()(hidden)
 hidden=Conv2D(64,(3,3), strides=1, padding =padding, activation = activation, name = 'conv5' )(hidden)
 hidden=BatchNormalization()(hidden)
 hidden=Conv2D(64,(5,5), strides=1, padding =padding, activation = activation, name = 'conv6' )(hidden)
 hidden=BatchNormalization()(hidden)
 hidden=MaxPool2D()(hidden)
 hidden=Dropout(0.3 )(hidden)
 hidden=Flatten()(hidden)
 hidden=Dense(250,activation = activation, name="Dense1" )(hidden)
 hidden=Dropout(0.3 )(hidden)
 output = Dense(classes, activation = "softmax" )(hidden)
 model = Model(inputs=input_layer, outputs=output)
 return model
optimizer = Adam(learning_rate=initial_learningrate, beta_1=0.9, beta_2=0.999, amsgrad=False )
!git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext"./ 
<import_modules>
model = build_model1(input_shape=(28, 28, 1), classes = 10)
 model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"] )
from apex import amp
import os
import cv2
import glob
import pydicom
import numpy as np
import pandas as pd
import torch
import torch.optim as optim
from albumentations import Compose, ShiftScaleRotate, Resize, CenterCrop, HorizontalFlip, RandomBrightnessContrast
from albumentations.pytorch import ToTensor
from torch.utils.data import Dataset
from tqdm import tqdm_notebook as tqdm
from matplotlib import pyplot as plt
from torchvision import transforms<load_from_csv>
datagen.fit(X_train)
 datagen.fit(X_val)
 callbacks = [checkpoint,learning_rate_reduction]
history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs,
validation_data=valid_datagen.flow(X_val,y_val),
verbose = 1,
callbacks = callbacks)
class IntracranialDataset(Dataset):
def __init__(self, csv_file, path, labels, transform=None):
self.path = path
self.data = pd.read_csv(csv_file)
 self.transform = transform
self.labels = labels
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join(self.path, self.data.loc[idx, 'Image'] + '.png')
 img = cv2.imread(img_name)
 if self.transform:
augmented = self.transform(image=img)
 img = augmented['image']
if self.labels:
labels = torch.tensor(
 self.data.loc[idx, ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']])
 return {'image': img, 'labels': labels}
else:
return {'image': img} 
<load_from_csv>
y_pre_test=model.predict(X_val)
 y_pre_test=np.argmax(y_pre_test,axis=1)
 y_test=np.argmax(y_val,axis=1)
 conf=confusion_matrix(y_test,y_pre_test)
 conf=pd.DataFrame(conf,index=range(0,10),columns=range(0,10))
train = pd.read_csv(os.path.join(dir_csv, 'stage_1_train.csv'))
test = pd.read_csv(os.path.join(dir_csv, 'stage_1_sample_submission.csv'))<feature_engineering>
print('out of {} samples, we got {} incorrect'.format(len(X_train), round(len(X_train)- history.history['accuracy'][-1] * len(X_train))))
train[['ID', 'Image', 'Diagnosis']] = train['ID'].str.split('_', expand=True)
 train = train[['Image', 'Diagnosis', 'Label']]
train.drop_duplicates(inplace=True)
 train = train.pivot(index='Image', columns='Diagnosis', values='Label' ).reset_index()
train['Image'] = 'ID_' + train['Image']
train.head()<count_values>
Y_pred = model.predict(X_val)
 Y_pred_classes = np.argmax(Y_pred,axis = 1)
 Y_true = np.argmax(y_val,axis = 1)
 errors =(Y_pred_classes - Y_true != 0)
 Y_pred_classes_errors = Y_pred_classes[errors]
Y_pred_errors = Y_pred[errors]
Y_true_errors = Y_true[errors]
X_val_errors = X_val[errors]
def display_errors(errors_index,img_errors,pred_errors, obs_errors):

 n = 0
nrows = 3
ncols = 3
fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True,figsize =(10,10))
fig.tight_layout()
for row in range(nrows):
for col in range(ncols):
error = errors_index[n]
ax[row,col].imshow(( img_errors[error] ).reshape(( 28,28)))
 ax[row,col].set_title("Predicted label :{}
True label :{}".format(pred_errors[error],obs_errors[error]))
n += 1
Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)
 true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))
delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors
sorted_dela_errors = np.argsort(delta_pred_true_errors)
 most_important_errors = sorted_dela_errors[-18:-9]
display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)
undersample_seed=0
train["any"].value_counts()<define_variables>
predictions = model.predict(test )
num_ill_patients = train[train["any"]==1].shape[0]
num_ill_patients<randomize_order>
predictions = predictions.argmax(axis = -1)
 predictions
<merge><EOS> 
submission = pd.DataFrame({ 'id': Id,
'label': predictions })
 submission.to_csv(path_or_buf ="submission.csv", index=False )