Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<load_pretrained>
def install(package):
subprocess.check_call([sys.executable, "-m", "pip","install",package])
 install(".. /input/fastremap/fastremap-1.10.2-cp37-cp37m-manylinux1_x86_64.whl")
 install(".. /input/fillvoids/fill_voids-2.0.0-cp37-cp37m-manylinux1_x86_64.whl")
 install(".. /input/finalmask")
 install("pydicom" )
class CustomResnet_Att(nn.Module):
def __init__(self, model_name='resnet50', weights_path=None):
assert model_name in('resnet50', 'resnet101', 'resnet152')
 super().__init__()
self.basic_net = pretrainedmodels.__dict__[model_name](pretrained=None)
 pretrained_dict = self.basic_net.load_state_dict(torch.load(weights_path))
model_dict = self.basic_net.state_dict()
state_dict = {}
for k, v in pretrained_dict.items() :
if k in model_dict.keys() :
state_dict[k] = v
model_dict.update(state_dict)
 self.basic_net.load_state_dict(model_dict)
 self.attention_cell= Attention_Cell()
self.regress = RegressionDense(1280)
 self.classfier = ClassfierDense(1280)
 def forward(self, x):
x = self.basic_net(x)
 x = self.attention_cell(x)
 x = x.squeeze(-1 ).squeeze(-1)
 pred_1 = self.regress(x)
 pred_2 = self.classfier(x)
 return pred_1,pred_2<normalization>
sns.set(style="whitegrid")
 sns.set_context("paper")
class ClassifierModule(nn.Sequential):
def __init__(self, n_features):
super().__init__(
 nn.BatchNorm1d(n_features),
nn.Dropout(0.5),
nn.Linear(n_features, n_features),
nn.PReLU() ,
nn.BatchNorm1d(n_features),
nn.Dropout(0.2),
nn.Linear(n_features, 1),
)<choose_model_class>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )
class CustomResNet(nn.Module):
def __init__(self, model_name='resnet50', weights_path=None):
assert model_name in('resnet50', 'resnet101', 'resnet152')
 super().__init__()
self.net = pretrainedmodels.__dict__[model_name](pretrained=None)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net.last_linear.in_features
self.net.avgpool = nn.AdaptiveAvgPool2d(1)
 self.net.last_linear = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )<choose_model_class>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
train=pd.read_csv(f"{ROOT}/train.csv")
 train.head()
class CustomResNeXt(nn.Module):
def __init__(self, model_name='resnext101_32x4d', weights_path=None):
assert model_name in('resnext101_32x4d', 'resnext101_64x4d')
 super().__init__()
self.net = pretrainedmodels.__dict__[model_name](pretrained=None)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net.last_linear.in_features
self.net.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.net.last_linear = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )<choose_model_class>
sample_submission=pd.read_csv(f"{ROOT}/sample_submission.csv")
 test=pd.read_csv(f"{ROOT}/test.csv")
 test.head()
class CustomSENet(nn.Module):
def __init__(self, model_name='se_resnet50', weights_path=None):
assert model_name in('senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152', 'se_resnext50_32x4d', 'se_resnext101_32x4d')
 super().__init__()
self.net = pretrainedmodels.__dict__[model_name](pretrained=None)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net.last_linear.in_features
self.net.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.net.last_linear = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )<choose_model_class>
train['Patient_Week']=train['Patient']+'_'+train['Weeks'].astype(str)
 lists=train['Patient_Week'][train.duplicated(['Patient_Week'], keep=False)].unique().tolist()
for patient_week in lists:
new_row=train.loc[train['Patient_Week']==patient_week].groupby(['Patient','Weeks','Age','Sex','SmokingStatus','Patient_Week'] ).mean().copy()
train=train[train['Patient_Week']!=patient_week]
train=train.append(new_row.reset_index())
 add=train.copy()
add.rename(columns={'Weeks':'base_weeks','FVC':'base_fvc'},inplace=True)
 final=train.merge(add,on='Patient')
 final.drop(['Patient_Week_x','Age_y','Sex_y','SmokingStatus_y','Percent_y'],axis=1,inplace=True)
 final.rename(columns={'Weeks':'base_week','FVC':'base_fvc','base_fvc':'FVC','Percent_x':'base_percent','Patient_Week_y':'Patient_Week','Age_x':'Age','Sex_x':'sex','SmokingStatus_x':'smokingstatus','base_weeks':'predict_week'},inplace=True)
 final['weeks_passed']=final['predict_week']-final['base_week']
cols=['Patient','Patient_Week', 'base_week', 'base_fvc', 'base_percent', 'Age', 'sex','smokingstatus','predict_week','weeks_passed', 'FVC']
final=final[cols]
final=final.loc[final['weeks_passed']!=0]
final.reset_index(drop=True,inplace=True)
 final.head()
class CustomEfficientNet(nn.Module):
def __init__(self, model_name='efficientnet-b0', weights_path=None):
assert model_name in('efficientnet-b0', 'efficientnet-b1', 'efficientnet-b2', 'efficientnet-b3', 'e fficientnet-b4', 'efficientnet-b5')
 super().__init__()
self.net = EfficientNet.from_name(model_name)
 self.net.load_state_dict(torch.load(weights_path))
n_features = self.net._fc.in_features
self.net._fc = ClassifierModule(n_features)
 def forward(self, x):
return self.net(x )<train_model>
test.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'},inplace=True)
 Week=sample_submission['Patient_Week'].apply(lambda x : x.split('_')[1] ).unique()
Week=np.tile(Week, len(test['Patient']))
test=test.loc[test.index.repeat(146)].reset_index(drop=True)
 test['predict_week']=Week
test['Patient_Week']=test['Patient']+'_'+test['predict_week']
test['weeks_passed']=test['predict_week'].astype(int)-test['base_Week'].astype(int)
 test.rename(columns={'base_Week':'base_week','base_FVC':'base_fvc','base_Percent':'base_percent','base_Age':'Age','Sex':'sex','SmokingStatus':'smokingstatus'},inplace=True)
 cols=['Patient','Patient_Week','base_week','base_fvc','base_percent','Age','sex','smokingstatus','predict_week','weeks_passed']
test=test[cols]
print(test.shape)
 test.head()
LOGGER.debug(f'Fold: {FOLD}')
 LOGGER.debug(f'Model: {MODEL}')
 LOGGER.debug(f'Train params: {train_params}' )<load_from_csv>
file_path= '.. /input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/10.dcm'
dataset = pydicom.dcmread(file_path)
with timer('Prepare train and valid sets'):
with timer(' * define dataset'):
train_dataset = MyDataset(train_df, transform =train_transform,train=True)
 valid_dataset = MyDataset(val_df, transform=train_transform,train =False)
 with timer(' * define dataloader'):
train_loader = DataLoader(train_dataset,
batch_size=train_params['train_batch_size'],
shuffle=True)
 valid_loader = DataLoader(valid_dataset,
batch_size=train_params['test_batch_size'],
shuffle=False)
 LOGGER.debug(f'train size: {len(train_dataset)}, valid size: {len(valid_dataset)}' )<compute_train_metric>
def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = abs(y_pred[:,2] - y_pred[:,0])
 fvc_pred = y_pred[:,1]
sigma_clip = tf.maximum(sigma, 70)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, 1000)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return metric
def qloss(y_true, y_pred):
qs = [0.8,0.5,0.2]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return kb.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def multi_loss(pred_reg,pred_classfier,y_true,multi_method='mse_ce',alpha = 1,beta =1):
assert multi_method in('mse_ce','')
 loss = 0;c1=0;c2=0
if multi_method =='mse_ce':
c1 = nn.MSELoss()
c2 = nn.CrossEntropyLoss()
loss_reg = c1(pred_reg,y_true)
 y_true = y_true.long()
loss_class = c2(pred_classfier,y_true.squeeze())
 loss = alpha*loss_reg + beta*loss_class
return loss<choose_model_class>
input_image = sitk.ReadImage('.. /input/osic-pulmonary-fibrosis-progression/train/ID00007637202177411956430/12.dcm')
 segmentation = mask.apply(input_image)
 plt.figure(figsize=(10,10))
plt.imshow(segmentation[0] )
<init_hyperparams>
atten=pd.read_csv('.. /input/attent-1/atten(1 ).csv')
 patients=os.listdir(f"{ROOT}/{how}")
 avg_atten_test=[]
for patient in patients:
try:
mid=mid_image_test(patient,True,True)
 postives=mid>mid.min()
mid[postives].mean()
avg_atten_test.append(mid[postives].mean())
 except:
avg_atten_test.append(np.nan)
 continue
. softmax(input, self.dim, _stacklevel=5)
 2020-07-02 02:27:20,329 DEBUG Epoch 1 - avg_train_loss: 0.3125 avg_val_loss: 4.7410 time: 112s
2020-07-02 02:27:20,330 DEBUG - qwk: 0.000000 coefficients: [0.5 1.5 2.5 3.5]
2020-07-02 02:29:02,828 DEBUG Epoch 2 - avg_train_loss: 0.3046 avg_val_loss: 5.4275 time: 102s
2020-07-02 02:29:02,830 DEBUG - qwk: 0.271479 coefficients: [-0.542408 2.047389 3.62589 5.926311]
2020-07-02 02:30:45,871 DEBUG Epoch 3 - avg_train_loss: 0.3203 avg_val_loss: 4.9421 time: 103s
2020-07-02 02:30:45,872 DEBUG - qwk: 0.060628 coefficients: [0.5 1.5 2.5 3.5]
2020-07-02 02:32:29,153 DEBUG Epoch 4 - avg_train_loss: 0.2932 avg_val_loss: 2.9958 time: 103s
2020-07-02 02:32:29,155 DEBUG - qwk: 0.602941 coefficients: [0.276694 1.188741 0.554224 7.733649]
2020-07-02 02:34:12,737 DEBUG Epoch 5 - avg_train_loss: 0.2489 avg_val_loss: 3.6941 time: 104s
2020-07-02 02:34:12,741 DEBUG - qwk: 0.364468 coefficients: [0.40734 0.359478 2.882061 4.309653]
2020-07-02 02:35:55,522 DEBUG Epoch 6 - avg_train_loss: 0.2240 avg_val_loss: 2.6687 time: 103s
2020-07-02 02:35:55,523 DEBUG - qwk: 0.607598 coefficients: [0.494824 0.815553 2.177532 4.449734]
2020-07-02 02:37:39,348 DEBUG Epoch 7 - avg_train_loss: 0.1965 avg_val_loss: 4.0795 time: 104s
2020-07-02 02:37:39,349 DEBUG - qwk: 0.653501 coefficients: [0.596621 1.498057 1.842185 4.625971]
2020-07-02 02:39:22,939 DEBUG Epoch 8 - avg_train_loss: 0.1642 avg_val_loss: 2.6657 time: 104s
2020-07-02 02:39:22,942 DEBUG - qwk: 0.713007 coefficients: [0.461684 1.598026 1.734047 4.394338]
2020-07-02 02:41:06,369 DEBUG Epoch 9 - avg_train_loss: 0.1579 avg_val_loss: 2.8600 time: 103s
2020-07-02 02:41:06,371 DEBUG - qwk: 0.635940 coefficients: [0.519184 1.490187 2.401873 3.643743]
2020-07-02 02:42:49,061 DEBUG Epoch 10 - avg_train_loss: 0.1606 avg_val_loss: 2.5784 time: 103s
2020-07-02 02:42:49,063 DEBUG - qwk: 0.563788 coefficients: [0.516132 0.831461 2.752982 3.96373 ]
2020-07-02 02:44:32,402 DEBUG Epoch 11 - avg_train_loss: 0.1453 avg_val_loss: 2.4899 time: 103s
2020-07-02 02:44:32,404 DEBUG - qwk: 0.559555 coefficients: [0.602128 0.988473 2.593635 3.840373]
2020-07-02 02:46:15,584 DEBUG Epoch 12 - avg_train_loss: 0.1299 avg_val_loss: 2.4178 time: 103s
2020-07-02 02:46:15,586 DEBUG - qwk: 0.618733 coefficients: [0.715129 1.184878 1.110287 4.461294]
2020-07-02 02:46:15,646 INFO [Train model] done in 1247 s .<import_modules>
final1=final.copy()
final1=final1.merge(atten,on='Patient')
 X1=final1[['base_fvc','base_percent','Age','sex','smokingstatus','weeks_passed','avg_atten','base_week']].copy()
y1=final1.FVC.copy()
import pandas as pd
import cv2
import time
import torchvision
import torch.nn as nn
from tqdm import tqdm_notebook as tqdm
import scipy as sp
from sklearn import metrics
from functools import partial
from PIL import Image, ImageFile
from torch.utils.data import Dataset
import torch
import torch.optim as optim
from torchvision import transforms
from torch.optim import lr_scheduler
import os
import numpy as np
from sklearn.model_selection import train_test_split, StratifiedKFold
from scipy import stats
import random
import albumentations
from albumentations import torch as AT<set_options>
enc = OneHotEncoder(handle_unknown='ignore')
 enc.fit(X1[['sex','smokingstatus']])
 encoded=pd.DataFrame(enc.transform(X1[['sex','smokingstatus']] ).toarray())
 X1=X1.join(encoded)
 X1.drop(['smokingstatus','sex'],axis=1,inplace=True)
 scaler=preprocessing.MinMaxScaler().fit(X1)
 X1=pd.DataFrame(scaler.transform(X1))
X1.head()
device = torch.device("cuda:0")
 ImageFile.LOAD_TRUNCATED_IMAGES = True<categorify>
atten_test=pd.DataFrame({'Patient':patients,'avg_atten':avg_atten_test})
 atten_test['avg_atten']=atten_test['avg_atten'].fillna(atten["avg_atten"].mean())
 X_test=test.merge(atten_test,on='Patient')
 X_test=X_test[['base_fvc','base_percent','Age','sex','smokingstatus','weeks_passed','avg_atten','base_week']].copy()
encoded=pd.DataFrame(enc.transform(X_test[['sex','smokingstatus']] ).toarray())
 X_test=X_test.join(encoded)
 X_test.drop(['smokingstatus','sex'],axis=1,inplace=True)
 X_test=pd.DataFrame(scaler.transform(X_test))
X1=X1.astype(np.float32)
 y1=y1.astype(np.float32 )
class RetinopathyDataset_v2(Dataset):
def __init__(self, df, transform, train=True):
self.data = df.reset_index()
if train:
self.prefix = "train"
else:
self.prefix = "test"
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join('.. /input/aptos2019-blindness-detection/{}_images'.format(self.prefix),
self.data.loc[idx, 'id_code'] + '.png')
 img = cv2.imread(img_name)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 image = self.transform(image=img)
 image = image['image']
label = torch.tensor(self.data.loc[idx, 'diagnosis'])
 return {'image': image,
'labels': label
}<load_from_csv>
inputs= keras.Input(shape=[11])
 dense = layers.Dense(100, activation="relu")
 x = dense(inputs)
 x = layers.Dense(100, activation="relu" )(x)
 output1 = layers.Dense(3,activation='linear' )(x)
 model = keras.Model(inputs=inputs, outputs=output1)
 model.summary()
class RetinopathyDataset(Dataset):
def __init__(self, csv_file, transform, train=True):
self.data = pd.read_csv(csv_file)
 if train:
self.prefix = "train"
else:
self.prefix = "test"
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join('.. /input/aptos2019-blindness-detection/{}_images'.format(self.prefix), self.data.loc[idx, 'id_code'] + '.png')
 img = cv2.imread(img_name)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 image = self.transform(image=img)
 image = image['image']
label = torch.tensor(self.data.loc[idx, 'diagnosis'])
 return {'image': image,
'labels': label
}<compute_test_metric>
model.compile(loss=mloss(0.8),optimizer='adam',metrics=score)
 model.fit(X1, y1,batch_size=512,epochs=130 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<choose_model_class>
preds_high=model.predict(X_test)[:,0]
preds_low=model.predict(X_test)[:,2]
preds=model.predict(X_test)[:,1]
model = torchvision.models.resnet101(pretrained=False)
 model.load_state_dict(torch.load(".. /input/fastai-pretrained-models/resnet101-5d3b4d8f.pth"))
for param in model.parameters() :
param.requires_grad = False
model.fc = nn.Sequential(
 nn.Linear(in_features=2048, out_features=1, bias=True), 
)
 model = model.to(device )<data_type_conversions>
preds_set=pd.DataFrame({'preds_high':preds_high})
 preds_set['preds']=preds
preds_set['preds_low']=preds_low
preds_set['sigma_pred']=abs(preds_set['preds_high']-preds_set['preds_low'])
 preds_set.reset_index(inplace=True,drop=True)
 preds_set
<load_from_csv><EOS> 
submission=pd.DataFrame({'Patient_Week':test['Patient_Week'],'FVC': preds_set['preds'],'Confidence':preds_set['sigma_pred']})
 submission['FVC']=submission['FVC'].apply(lambda x: round(x, 4)) /1
submission['Confidence']=submission['Confidence'].apply(lambda x: round(x, 4))
submission.to_csv('submission.csv',index=False)
 submission.head()