Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<set_options>
train = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test = pd.read_csv('.. /input/Kannada-MNIST/test.csv')
 train.columns
warnings.filterwarnings("ignore" )<import_modules>
sample_size = 8000
np.random.seed(2018)
 idx = np.random.choice(60000, size=sample_size, replace=False)
 train_sample = train.loc[idx,'pixel0':]
label_sample = train.loc[idx,'label']
train_sample
embeddings = TSNE(verbose=2 ).fit_transform(train_sample )
import torch.nn as nn
import torch
from torch.utils.data.dataset import Dataset
from torch.utils.data import DataLoader<set_options>
y_train = train['label']
y_train
def seed_everything(seed=2020):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = True
seed_everything(42 )<define_variables>
train['label'].value_counts()
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
device = torch.device('cuda' )<load_from_csv>
g = sns.countplot(y_train)
 y_train.value_counts()
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep' )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
X = train_data.values / 255.0
X_test = test.loc[:,'pixel0':].values / 255.0
y = train_label
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
X_train,X_val,y_train,y_val = train_test_split(X,y,test_size = 0.1 )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
y_train = to_categorical(y_train)
 y_val = to_categorical(y_val )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
def DL_Model(filter1_size=64, filter2_size=32 , activation='relu', optimizer='Adam', padding='Same'):
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(filter1_size,(3,3),padding = padding,activation = activation,
input_shape =(28,28,1)))
 model.add(tf.keras.layers.Conv2D(filter1_size,(3,3),padding = padding,activation=activation))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.MaxPool2D(pool_size=(2,2)))
 model.add(tf.keras.layers.Conv2D(filter2_size,(3,3),padding = padding,activation=activation))
model.add(tf.keras.layers.Conv2D(filter2_size,(3,3),padding = padding,activation=activation))
model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.MaxPool2D(pool_size=(2,2)))
 model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(128,activation=activation))
model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(256,activation=activation))
model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Dense(10,activation='softmax'))
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 return model
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
filter1_size = [64]
filter2_size = [64]
activation = ['relu','tanh','sigmoid']
padding = ['Same']
optimizer = ['Adam','RMSprop']
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
<feature_engineering>
model=DL_Model(64,64,'relu','Adam','Same' )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
epochs = 40
batch_size = 128
model.fit(X_train,y_train,
validation_data=(X_val,y_val),
epochs=epochs,
batch_size=batch_size )
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data<feature_engineering>
model.evaluate(X_val,y_val )
class MishFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, x):
ctx.save_for_backward(x)
 return x * torch.tanh(F.softplus(x))
@staticmethod
def backward(ctx, grad_output):
x = ctx.saved_variables[0]
sigmoid = torch.sigmoid(x)
 tanh_sp = torch.tanh(F.softplus(x))
return grad_output *(tanh_sp + x * sigmoid *(1 - tanh_sp * tanh_sp))
class Mish(nn.Module):
def forward(self, x):
return MishFunction.apply(x)
 def to_Mish(model):
for child_name, child in model.named_children() :
if isinstance(child, nn.ReLU):
setattr(model, child_name, Mish())
 else:
to_Mish(child )<compute_test_metric>
model=DL_Model(64,64,'relu','Adam','Same' )
C1, C2 = torch.tensor(70,dtype=torch.float),torch.tensor(1000,dtype=torch.float)
 C1, C2 = C1.to(device),C2.to(device)
 def score(y_true, y_pred):
y_true = y_true.to(torch.float)
 y_pred = y_pred.to(torch.float)
 sigma = y_pred[:,2] - y_pred[:,0]
fvc_pred = y_pred[:,1]
sigma_clip = torch.max(sigma, C1)
 delta = torch.abs(y_true[:,0] - fvc_pred)
 delta = torch.min(delta, C2)
 sq2 = torch.sqrt(torch.tensor(2, dtype=torch.float))
metric =(delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)
 return torch.mean(metric)
 def qloss(y_true, y_pred):
device = y_true.device
qs = [0.2, 0.50, 0.8]
q = torch.tensor(np.array([qs]), dtype=torch.float32)
 q = q.to(device)
 e = y_true - y_pred
v = torch.max(q*e,(q-1)*e)
 return torch.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
class make_model(nn.Module):
def __init__(self, in_ch, out_ch=3):
super(make_model, self ).__init__()
self.fc1 = nn.Sequential(
 nn.Linear(in_ch, 100),
Mish() 
)
 self.fc2 = nn.Sequential(
 nn.Linear(100, 100),
Mish() 
)
 self.fc3_p1 = nn.Linear(100, out_ch)
 self.fc3_p2 = nn.Sequential(
 nn.Linear(100, out_ch),
Mish() 
)
 def forward(self, x):
x = self.fc1(x)
 x = self.fc2(x)
 x1 = self.fc3_p1(x)
 x2 = self.fc3_p2(x)
 x = x1 + torch.cumsum(x2,dim=1)
 return x 
<compute_test_metric>
early_stopping_monitor = tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=6,
verbose=0,
mode='auto',
baseline=None,
restore_best_weights=True
)
Mloss = mloss(0.8)
 y_true =(torch.tensor([2100,2300])).to(device ).reshape(2,1)
 pred =(torch.tensor([[1800,2100,2400],
[2100,2300,2600]])).to(device ).reshape(2,3)
 Mloss(y_true,pred )<feature_engineering>
learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
NFOLD = 5
kf = KFold(n_splits=NFOLD)
 pd_patient = pd.DataFrame({"Patient":tr["Patient"].unique() })
 for idx,(tr_idx, val_idx)in enumerate(kf.split(pd_patient)) :
pd_patient.loc[val_idx,"fold"] = idx
pd_patient.head()<feature_engineering>
epochs = 30
batch_size = 86
tr['fold'] = -1
for i in range(len(pd_patient)) :
tr.fold[tr.Patient==pd_patient.loc[i,"Patient"]] = pd_patient.loc[i,"fold"]
tr.head()<categorify>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
ze =(sub[FE].values ).astype(np.float32)
 pe = np.zeros(( ze.shape[0], 3))
batch = 128<prepare_x_and_y>
history = model.fit(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )
class Data_Generate(Dataset):
def __init__(self,data,label=None):
self.data = data
self.label = label
def __getitem__(self,index):
z_ = self.data[index]
if self.label is not None:
y_ = self.label[index]
y_ = y_[None,]
return z_,y_
else:
return z_
def __len__(self):
return len(self.data )<load_pretrained>
model.evaluate(X_val,y_val )
test_db = Data_Generate(ze)
 test_loader = DataLoader(test_db, batch_size=batch, shuffle=False, num_workers=4 )<train_model>
y_val_predicted = model.predict_classes(X_val)
 y_val_actual=np.argmax(y_val, axis=-1)
 cm = confusion_matrix(y_val_actual, y_val_predicted )
class EarlyStopping:

 def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
self.delta = delta
self.path = path
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score + self.delta:
self.counter += 1
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , self.path)
 self.val_loss_min = val_loss<data_type_conversions>
result = model.predict_classes(X_test )
%%time
cnt = 0
EPOCHS = 1000
criterion = mloss(0.8)
 list_train_loss,list_val_loss,list_train_score,list_val_score = [],[],[],[]
for fold in range(NFOLD):
val_out = []
print(f"FOLD {fold+1}")
 tr_z = tr[FE][tr.fold!=fold].values.astype(np.float32)
 tr_y = tr.FVC[tr.fold!=fold].values.astype(np.float32)
 val_z = tr[FE][tr.fold==fold].values.astype(np.float32)
 val_y = tr.FVC[tr.fold==fold].values.astype(np.float32)
 train_db = Data_Generate(tr_z,tr_y)
 train_loader = DataLoader(train_db, batch_size=batch, shuffle=True, num_workers=4)
 val_db = Data_Generate(val_z,val_y)
 val_loader = DataLoader(val_db, batch_size=batch, shuffle=False, num_workers=4)
 tr_num_batch = len(train_loader)
 val_num_batch = len(val_loader)
 net = make_model(in_ch=len(FE)).to(device)
 optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=0.001, weight_decay=5e-4)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=20, min_lr=1e-8, verbose=False)
 early_stopping = EarlyStopping(patience=80,path=f'Osic-NN-fold_{fold}.pth')
 for epoch in tqdm(range(EPOCHS)) :
train_loss,train_score,val_loss,val_score = 0,0,0,0
net.train()
for idx, sample in enumerate(train_loader):
data, label = sample
data, label = data.to(device), label.to(device)
 out = net(data)
 loss = criterion(label, out)
 score_ = score(label ,out)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss += loss.item() /tr_num_batch
train_score += score_.item() /tr_num_batch
list_train_loss.append(train_loss)
 list_train_score.append(train_score)
 net.eval()
for idx, sample in enumerate(val_loader):
data, label = sample
data, label = data.to(device), label.to(device)
 with torch.no_grad() :
out = net(data)
 val_out.append(out.cpu().numpy())
 loss = criterion(label, out)
 score_ = score(label, out)
 val_loss += loss.item() /val_num_batch
val_score += score_.item() /val_num_batch
list_val_loss.append(val_loss)
 list_val_score.append(val_score)
 early_stopping(val_loss, net)
 if early_stopping.early_stop:
print("Early stopping")
 break
scheduler.step(val_loss)
 print(f"train loss: {train_loss} train score: {train_score}
\
val loss: {val_loss} val score: {val_score}
\
final lr: {optimizer.param_groups[0]['lr']}" 
)
<predict_on_test>
sub_df = test[['id']]
sub_df['label'] = result
<feature_engineering><EOS> 
sub_df.to_csv('submission.csv',index=False )