Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<compute_test_metric>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
def weighted_mse_loss(preds, target, weight):
return torch.sum(weight *(preds - target)** 2 )<choose_model_class>
exclude_test_patient_data_from_trainset = True
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 if exclude_test_patient_data_from_trainset:
train = train[~train['Patient'].isin(test['Patient'].unique())]
train = pd.concat([train, test], axis=0, ignore_index=True)\ 
.drop_duplicates()
le_id = LabelEncoder()
train['PatientID'] = le_id.fit_transform(train['Patient'])
 train.head()
data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4)
 data_loader_val = torch.utils.data.DataLoader(validation_dataset, batch_size=32, shuffle=True, num_workers=4)
 num_epochs = 25
optimizer = optim.Adam(model.parameters() , lr=0.0002)
 alpha_mixup = 0.3
n_freeze = 2
weight_pow = 0.3
label_counts = train_df['diagnosis'].value_counts().values
label_weights =(label_counts[:] / label_counts.sum())**(-weight_pow)
 label_weights /= label_weights[0]
print(label_weights )<choose_model_class>
import pymc3 as pm
since = time.time()
criterion = weighted_mse_loss
best = 0
nep = 0
best_coeff = []
month_day = time.gmtime() [1:3]
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
 model.train()
running_loss = 0.0
tk0 = tqdm(data_loader, total=int(len(data_loader)))
 counter = 0
if epoch == n_freeze:
for param in model.parameters() :
param.requires_grad = True
for bi, d in enumerate(tk0):
inputs = d["image"]
labels = d["labels"].view(-1, 1)
 weights = labels[:, 0].clone()
for i in range(len(label_weights)) :
weights[labels[:, 0] == i] = label_weights[i]
shuffled_index = list(range(inputs.shape[0]))
random.shuffle(shuffled_index)
 mixed_up_inputs = inputs[shuffled_index, :, :, :]
mixed_up_labels = labels[shuffled_index, :]
mixed_up_weights = mixed_up_labels[:, 0].clone()
for i in range(len(label_weights)) :
mixed_up_weights[mixed_up_labels[:, 0] == i] = label_weights[i]
l_mixup = stats.beta.rvs(a=alpha_mixup, b=alpha_mixup, size=inputs.shape[0])
 inputs = torch.as_tensor(l_mixup, dtype=torch.float ).view(-1, 1, 1, 1)* inputs
inputs += torch.as_tensor(1.- l_mixup, dtype=torch.float ).view(-1, 1, 1, 1)* mixed_up_inputs
labels = labels.float()
mixed_up_labels = mixed_up_labels.float()
labels = torch.as_tensor(l_mixup, dtype=torch.float ).view(-1, 1)* labels
labels += torch.as_tensor(1.- l_mixup, dtype=torch.float ).view(-1, 1)* mixed_up_labels
weights = weights.float()
mixed_up_weights = mixed_up_weights.float()
weights = torch.as_tensor(l_mixup, dtype=torch.float ).view(-1, 1)* weights
weights += torch.as_tensor(1.- l_mixup, dtype=torch.float ).view(-1, 1)* mixed_up_weights
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.float)
 weights = weights.to(device, dtype=torch.float)
 optimizer.zero_grad()
with torch.set_grad_enabled(True):
outputs = model(inputs)
 loss = criterion(outputs, labels, weights)
 loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 counter += 1
tk0.set_postfix(loss=(running_loss /(counter * data_loader.batch_size)))
 epoch_loss = running_loss / len(data_loader)
 print('Training Loss: {:.4f}'.format(epoch_loss))
model.eval()
counter = 0
val_loss = 0.0
preds = []
full_labels = []
for d in data_loader_val:
inputs = d["image"]
labels = d["labels"].view(-1, 1)
 inputs = inputs.to(device, dtype=torch.float)
 with torch.set_grad_enabled(False):
outputs = model(inputs)
 preds.append(outputs.cpu().numpy())
 full_labels.append(labels.numpy())
 preds = np.vstack(preds)[:, 0]
full_labels = np.vstack(full_labels)[:, 0]
optR = OptimizedRounder()
optR.fit(preds, full_labels)
 coefficients = optR.coefficients()
valid_predictions = optR.predict(preds, coefficients)
 val_loss = metrics.cohen_kappa_score(valid_predictions, full_labels, weights='quadratic')
 print('Validation Loss: {:.4f}'.format(val_loss))
if val_loss > best:
best = val_loss
nep = epoch
best_coeff = coefficients[:]
print('Current best', best, nep, best_coeff)
 torch.save(model.state_dict() , "model_best.pth")
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))<find_best_params>
n_patients = train['Patient'].nunique()
FVC_obs = train['FVC'].values
Weeks = train['Weeks'].values
PatientID = train['PatientID'].values
if best_coeff[-1] < 3.0:
best_coeff[-1] = 3.0<load_pretrained>
pred_template = []
for i in range(train['Patient'].nunique()):
df = pd.DataFrame(columns=['PatientID', 'Weeks'])
 df['Weeks'] = np.arange(-12, 134)
 df['PatientID'] = i
pred_template.append(df)
 pred_template = pd.concat(pred_template, ignore_index=True )
del inputs, labels, weights, mixed_up_inputs, mixed_up_labels, mixed_up_weights, outputs
del train_dataset, train_df, validation_dataset, validation_df
torch.cuda.empty_cache()<load_pretrained>
with model_a:
pm.set_data({
"PatientID_shared": pred_template['PatientID'].values.astype(int),
"Weeks_shared": pred_template['Weeks'].values.astype(int),
"FVC_obs_shared": np.zeros(len(pred_template)).astype(int),
})
 post_pred = pm.sample_posterior_predictive(trace_a )
model.load_state_dict(torch.load("model_best.pth"))
model.eval()<define_variables>
df = pd.DataFrame(columns=['Patient', 'Weeks', 'FVC_pred', 'sigma'])
 df['Patient'] = le_id.inverse_transform(pred_template['PatientID'])
 df['Weeks'] = pred_template['Weeks']
df['FVC_pred'] = post_pred['FVC_like'].T.mean(axis=1)
 df['sigma'] = post_pred['FVC_like'].T.std(axis=1)
 df['FVC_inf'] = df['FVC_pred'] - df['sigma']
df['FVC_sup'] = df['FVC_pred'] + df['sigma']
df = pd.merge(df, train[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 df = df.rename(columns={'FVC': 'FVC_true'})
 df.head()
n_tta = 10
bsize = 4<categorify>
tr = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df = pd.merge(df, tr[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 use_only_last_3_measures = True
if use_only_last_3_measures:
y = df.dropna().groupby('Patient' ).tail(3)
 else:
y = df.dropna()
rmse =(( y['FVC_pred'] - y['FVC'])** 2 ).mean() **(1/2)
 print(f'RMSE: {rmse:.1f} ml')
 sigma_c = y['sigma'].values
sigma_c[sigma_c < 70] = 70
delta =(y['FVC_pred'] - y['FVC'] ).abs()
delta[delta > 1000] = 1000
lll = - np.sqrt(2)* delta / sigma_c - np.log(np.sqrt(2)* sigma_c)
 print(f'Laplace Log Likelihood: {lll.mean() :.4f}' )
test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=bsize, shuffle=False, num_workers=4)
 test_preds = np.zeros(( len(test_dataset), 1))
for j in range(n_tta):
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds[i * bsize:(i + 1)* bsize] += pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_preds /= n_tta<save_to_csv>
train['Male'] = train['Sex'].apply(lambda x: 1 if x == 'Male' else 0)
 train["SmokingStatus"] = train["SmokingStatus"].astype(
 pd.CategoricalDtype(['Ex-smoker', 'Never smoked', 'Currently smokes'])
)
 aux = pd.get_dummies(train["SmokingStatus"], prefix='ss')
 aux.columns = ['ExSmoker', 'NeverSmoked', 'CurrentlySmokes']
train['ExSmoker'] = aux['ExSmoker']
train['CurrentlySmokes'] = aux['CurrentlySmokes']
aux = train[['Patient', 'Weeks', 'Percent']].sort_values(by=['Patient', 'Weeks'])
 aux = train.groupby('Patient' ).head(1)
 aux = aux.rename(columns={'Percent': 'Percent_base'})
 train = pd.merge(train, aux[['Patient', 'Percent_base']], how='left',
on='Patient')
 train.head()
for i, pred in enumerate(test_preds):
if pred < best_coeff[0]:
test_preds[i] = 0
elif pred >= best_coeff[0] and pred < best_coeff[1]:
test_preds[i] = 1
elif pred >= best_coeff[1] and pred < best_coeff[2]:
test_preds[i] = 2
elif pred >= best_coeff[2] and pred < best_coeff[3]:
test_preds[i] = 3
else:
test_preds[i] = 4
sample = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 sample.diagnosis = test_preds.astype(int)
 sample.to_csv("submission.csv", index=False )<set_options>
n_patients = train['Patient'].nunique()
FVC_obs = train['FVC'].values
X = train[['Weeks', 'Male', 'ExSmoker', 'CurrentlySmokes',
'Percent_base']].values
PatientID = train['PatientID'].values
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
with pm.Model() as model_b:
FVC_obs_shared = pm.Data("FVC_obs_shared", FVC_obs)
 X_shared = pm.Data('X_shared', X)
 PatientID_shared = pm.Data('PatientID_shared', PatientID)
 mu_a = pm.Normal('mu_a', mu=1700, sigma=400)
 sigma_a = pm.HalfNormal('sigma_a', 1000.)
mu_b = pm.Normal('mu_b', mu=-4., sigma=1., shape=X.shape[1])
 sigma_b = pm.HalfNormal('sigma_b', 5.)
a = pm.Normal('a', mu=mu_a, sigma=sigma_a, shape=n_patients)
 b = pm.Normal('b', mu=mu_b, sigma=sigma_b, shape=(n_patients, X.shape[1]))
sigma = pm.HalfNormal('sigma', 150.)
FVC_est = a[PatientID_shared] +(b[PatientID_shared] * X_shared ).sum(axis=1)
 FVC_like = pm.Normal('FVC_like', mu=FVC_est,
sigma=sigma, observed=FVC_obs_shared )
import fastai
from fastai import *
from fastai.vision import *
from fastai.callbacks import *
import cv2
import pandas as pd
import matplotlib.pyplot as plt<set_options>
aux = train.groupby('Patient' ).first().reset_index()
pred_template = []
for i in range(train['Patient'].nunique()):
df = pd.DataFrame(columns=['PatientID', 'Weeks'])
 df['Weeks'] = np.arange(-12, 134)
 df['PatientID'] = i
df['Male'] = aux[aux['PatientID'] == i]['Male'].values[0]
df['ExSmoker'] = aux[aux['PatientID'] == i]['ExSmoker'].values[0]
df['CurrentlySmokes'] = aux[aux['PatientID'] == i]['CurrentlySmokes'].values[0]
df['Percent_base'] = aux[aux['PatientID'] == i]['Percent_base'].values[0]
pred_template.append(df)
 pred_template = pd.concat(pred_template, ignore_index=True )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 1667
seed_everything(SEED )<feature_engineering>
with model_b:
pm.set_data({
"PatientID_shared": pred_template['PatientID'].values.astype(int),
"X_shared": pred_template[['Weeks', 'Male', 'ExSmoker',
'CurrentlySmokes',
'Percent_base']].values.astype(int),
"FVC_obs_shared": np.zeros(len(pred_template)).astype(int),
})
 post_pred = pm.sample_posterior_predictive(trace_b )
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head(10 )<define_variables>
df = pd.DataFrame(columns=['Patient', 'Weeks', 'FVC_pred', 'sigma'])
 df['Patient'] = le_id.inverse_transform(pred_template['PatientID'])
 df['Weeks'] = pred_template['Weeks']
df['FVC_pred'] = post_pred['FVC_like'].T.mean(axis=1)
 df['sigma'] = post_pred['FVC_like'].T.std(axis=1)
 df['FVC_inf'] = df['FVC_pred'] - df['sigma']
df['FVC_sup'] = df['FVC_pred'] + df['sigma']
df = pd.merge(df, train[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 df = df.rename(columns={'FVC': 'FVC_true'})
 df.head()
bs=32
src =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.2, seed=42)
.label_from_df(cols='diagnosis',label_cls=FloatList)
)
 src<normalization>
tr = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 df = pd.merge(df, tr[['Patient', 'Weeks', 'FVC']], how='left', on=['Patient', 'Weeks'])
 use_only_last_3_measures = True
if use_only_last_3_measures:
y = df.dropna().groupby('Patient' ).tail(3)
 else:
y = df.dropna()
rmse =(( y['FVC_pred'] - y['FVC_true'])** 2 ).mean() **(1/2)
 print(f'RMSE: {rmse:.1f} ml')
 sigma_c = y['sigma'].values
sigma_c[sigma_c < 70] = 70
delta =(y['FVC_pred'] - y['FVC_true'] ).abs()
delta[delta > 1000] = 1000
lll = - np.sqrt(2)* delta / sigma_c - np.log(np.sqrt(2)* sigma_c)
 print(f'Laplace Log Likelihood: {lll.mean() :.4f}' )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.10, max_zoom=1.3, max_warp=0.0, max_lighting=0.2)
 data =(
 src.transform(tfms,size=128)
.databunch(bs=bs)
.normalize(imagenet_stats)
)
 data<compute_test_metric>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 train = pd.concat([train, test], axis=0, ignore_index=True)\ 
.drop_duplicates()
le_id = LabelEncoder()
train['PatientID'] = le_id.fit_transform(train['Patient'] )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )<choose_model_class>
n_patients = train['Patient'].nunique()
FVC_obs = train['FVC'].values
Weeks = train['Weeks'].values
PatientID = train['PatientID'].values
with pm.Model() as model_a:
FVC_obs_shared = pm.Data("FVC_obs_shared", FVC_obs)
 Weeks_shared = pm.Data('Weeks_shared', Weeks)
 PatientID_shared = pm.Data('PatientID_shared', PatientID)
 mu_a = pm.Normal('mu_a', mu=1700., sigma=400)
 sigma_a = pm.HalfNormal('sigma_a', 1000.)
mu_b = pm.Normal('mu_b', mu=-4., sigma=1)
 sigma_b = pm.HalfNormal('sigma_b', 5.)
a = pm.Normal('a', mu=mu_a, sigma=sigma_a, shape=n_patients)
 b = pm.Normal('b', mu=mu_b, sigma=sigma_b, shape=n_patients)
 sigma = pm.HalfNormal('sigma', 150.)
FVC_est = a[PatientID_shared] + b[PatientID_shared] * Weeks_shared
FVC_like = pm.Normal('FVC_like', mu=FVC_est,
sigma=sigma, observed=FVC_obs_shared)
 trace_a = pm.sample(2000, tune=2000, target_accept=.9, init="adapt_diag" )
learn = cnn_learner(data, base_arch=models.resnet50 ,metrics=[quadratic_kappa],model_dir='/kaggle',pretrained=True,
callback_fns=[partial(EarlyStoppingCallback, monitor='quadratic_kappa', min_delta=0.01, patience=3)] )<train_model>
pred_template = []
for p in test['Patient'].unique() :
df = pd.DataFrame(columns=['PatientID', 'Weeks'])
 df['Weeks'] = np.arange(-12, 134)
 df['Patient'] = p
pred_template.append(df)
 pred_template = pd.concat(pred_template, ignore_index=True)
 pred_template['PatientID'] = le_id.transform(pred_template['Patient'])
 with model_a:
pm.set_data({
"PatientID_shared": pred_template['PatientID'].values.astype(int),
"Weeks_shared": pred_template['Weeks'].values.astype(int),
"FVC_obs_shared": np.zeros(len(pred_template)).astype(int),
})
 post_pred = pm.sample_posterior_predictive(trace_a )
<save_model><EOS> 
df = pd.DataFrame(columns=['Patient', 'Weeks', 'Patient_Week', 'FVC', 'Confidence'])
 df['Patient'] = pred_template['Patient']
df['Weeks'] = pred_template['Weeks']
df['Patient_Week'] = df['Patient'] + '_' + df['Weeks'].astype(str)
 df['FVC'] = post_pred['FVC_like'].T.mean(axis=1)
 df['Confidence'] = post_pred['FVC_like'].T.std(axis=1)
 final = df[['Patient_Week', 'FVC', 'Confidence']]
final.to_csv('submission.csv', index=False)
 print(final.shape)
 final.head()