Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<save_to_csv>
!pip install.. /input/pytorchcv/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet
submission_logreg = pd.DataFrame({
"ID_code": df_test["ID_code"],
"target": y_pred_logreg
})
 submission_logreg.to_csv('submission_logreg.csv', index=False )<import_modules>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(42 )
import logging, os, pandas as pd, numpy as np
import matplotlib.pyplot as plt, seaborn as sns
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.metrics import roc_auc_score, roc_curve<load_from_csv>
def load_scan(path):
paths = glob.glob(path+'/*.dcm')
 paths.sort(key=lambda f: int(re.sub('\D', '', f)))
 if len(paths)< 51:
paths = paths[:50]
slices = [pydicom.read_file(p)for p in paths]
try:
slices.sort(key=lambda x: float(x.ImagePositionPatient[2]))
except:
pass
image = np.stack([s.pixel_array.astype(float)for s in slices])
 return image, slices[0], slices
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_values>
IN_TRAIN = '.. /input/osic-pulmonary-fibrosis-progression/test/'
paths = os.listdir(IN_TRAIN)
 paths.sort()
sns.countplot(train['target'])
 print(train.target.value_counts(normalize=True))<count_duplicates>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv')
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient")
 sub['WHERE'] = 'test'
test['WHERE'] = 'sub'
train['WHERE'] = 'train'
data = train.append([test, sub] )
def dupl_max_df(df):
dupl_max = []
for feature in features:
values = df[feature].value_counts()
dupl_max.append([feature, values.max() , values.idxmax() ])
 dupl_max = pd.DataFrame(dupl_max, columns=['feature', '
return dupl_max<count_duplicates>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 COLS = ['Sex','SmokingStatus']
FE = []
FE += ['Patient']
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 FE += ['age','percent','week','BASE', 'WHERE']
meta_df = data[FE]
meta_df['fold'] = 0
meta_df = meta_df[meta_df.WHERE=='test']
meta_df.head()
dupl_max_df(train )<concatenate>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector)
 targets = []
tab = []
P = []
for i, p in enumerate(test.Patient.unique()):
sub = test.loc[test.Patient == p, :]
tab.append(get_tab(sub))
patients = pd.DataFrame({'Patient': test.Patient.unique() , 'target': 0, 'meta': tab, 'fold': 0})
 patients.head()
dupl_max_df(test )<feature_engineering>
train['fold'] = 0
%%time
for df in [train, test]:
df['sum'] = df[features].sum(axis=1)
 df['min'] = df[features].min(axis=1)
 df['max'] = df[features].max(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['std'] = df[features].std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurtosis(axis=1)
 df['med'] = df[features].median(axis=1 )<init_hyperparams>
class FCN(torch.nn.Module):
def __init__(self, base, in_f, num_classes, in_meta, dropout=True):
super(FCN, self ).__init__()
self.base = base
self.after_model = nn.Sequential(
 nn.Flatten() ,
nn.Dropout(0.5)
)
 self.meta = nn.Sequential(
 nn.Linear(in_meta, 100),
nn.ReLU() ,
nn.Linear(100, 100),
nn.ReLU() , 
)
 self.classification_meta = nn.Sequential(
 nn.Linear(in_f+100, num_classes)
)
 self.classification = nn.Sequential(
 nn.Linear(in_f, num_classes)
)
 self.meta_head = nn.Linear(100, num_classes)
 def forward(self, x, meta):
x = self.base(x)
 x = self.after_model(x)
 meta = self.meta(meta)
 features = torch.cat(( x,meta),dim=1)
 x = self.classification_meta(features)
 return x
def create_model(name, path=None):
model = get_model(name, pretrained=False)
 try:
features = list(model.children())[-1].in_features
except:
features = list(model.children())[-1][-1].in_features
model = nn.Sequential(*list(model.children())[:-1])
 model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))
model = FCN(model, features, config.n_seg_classes, config.meta_features, dropout=True)
 if path:
print('loading pretrained model {}'.format(path))
pretrain = torch.load(path)['model_state']
model.load_state_dict(pretrain)
 return model
features = [c for c in train.columns.values if c not in ['ID_code', 'target']]
target = train['target']
params = {
'bagging_freq': 2,
'bagging_fraction': 0.8,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.1403,
'learning_rate': 0.07,
'max_depth': 7,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 19,
'num_leaves': 14,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1,
'lambda_l1': 1.7916,
'lambda_l2': 4.7454,
'min_gain_to_split': 0.0319
}<split>
class FCN2(torch.nn.Module):
def __init__(self, base, in_f, num_classes, in_meta, dropout=True):
super(FCN2, self ).__init__()
self.base = base
self.after_model = nn.Sequential(
 nn.Flatten() ,
nn.Dropout(0.5)
)
 self.meta = nn.Sequential(
 nn.Linear(in_meta, 1024),
nn.ReLU() ,
nn.Linear(1024, 512),
nn.ReLU() , 
)
 self.classification_meta = nn.Sequential(
 nn.Linear(in_f+512, num_classes)
)
 self.classification = nn.Sequential(
 nn.Linear(in_f, num_classes)
)
 self.meta_head = nn.Linear(100, num_classes)
 def forward(self, x, meta):
x = self.base(x)
 x = self.after_model(x)
 meta = self.meta(meta)
 features = torch.cat(( x,meta),dim=1)
 x = self.classification_meta(features)
 return x
def create_model2(name, path=None):
model = get_model(name, pretrained=False)
 try:
features = list(model.children())[-1].in_features
except:
features = list(model.children())[-1][-1].in_features
model = nn.Sequential(*list(model.children())[:-1])
 model[0].final_pool = nn.Sequential(nn.AdaptiveAvgPool2d(1))
model = FCN2(model, features, config.n_seg_classes, config.meta_features, dropout=True)
 if path:
print('loading pretrained model {}'.format(path))
pretrain = torch.load(path)['model_state']
model.load_state_dict(pretrain)
 return model
%%time
val_size = 0.3
X_train, X_val, y_train, y_val = train_test_split(train[features], train['target'], test_size = val_size, random_state=42)
 iterations = 100
val_pred = np.zeros([int(len(train)*val_size), len(features)])
 test_pred = np.zeros([200000, len(features)])
 i = 0
for feature in features:
print(feature)
 feat = [feature]
train_data = lgb.Dataset(X_train[feat], y_train)
 gbm = lgb.train(params, train_data, iterations, verbose_eval=-1)
 val_pred[:, i] = gbm.predict(X_val[feat], num_iteration=gbm.best_iteration)
 test_pred[:, i] = gbm.predict(test[feat], num_iteration=gbm.best_iteration)
 i += 1
score = roc_auc_score(y_val,(val_pred ).sum(axis=1))
print('CV score: ', score )<split>
class config:
input_D = 30
input_H = 512
input_W = 512
n_seg_classes = 3
meta_features = 9
seed = 42
quantiles =(0.2, 0.5, 0.8)
 nets=[]
model_name = 'resnet50'
model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld1-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld2-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld3-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld4-2(2 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld5-2(2 ).pth')
 model.cuda()
nets.append(model)
 model_name = 'resnet50'
model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld1-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld2-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld3-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld4-2(4 ).pth')
 model.cuda()
nets.append(model)
 model = create_model(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld5-2(4 ).pth')
 model.cuda()
nets.append(model)
 model_name = 'resnet50'
model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld1-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld2-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld3-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld4-2(5 ).pth')
 model.cuda()
nets.append(model)
 model = create_model2(model_name,
path = '.. /input/osic-5fold-models2/resnet3d-fld5-2(5 ).pth')
 model.cuda()
nets.append(model )
%%time
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state=42)
 oof = np.zeros(len(train))
pred = np.zeros(len(test))
feature_important_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train, train['target'])) :
print('Fold: {}'.format(fold_))
trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 100000
clf = lgb.train(params, trn_data, num_round, valid_sets=[trn_data, val_data], verbose_eval=1000, early_stopping_rounds=3000)
 oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_important_df = pd.DataFrame()
fold_important_df['feature'] = features
fold_important_df['importance'] = clf.feature_importance()
fold_important_df['fold'] = fold_ + 1
feature_important_df = pd.concat([feature_important_df, fold_important_df], axis=0)
 pred = clf.predict(test[features], num_iteration=clf.best_iteration)/folds.n_splits
print('CV score: {:.5f}'.format(roc_auc_score(target, oof)) )<save_to_csv>
transform = A.Compose([
A.HorizontalFlip(p=1.0)
 ])
 transform2 = A.Compose([
A.VerticalFlip(p=1.0)
 ] )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = pred
submission.to_csv('submission.csv', index=False )<set_options>
def convertToHU(image, data):
image =(image + data.RescaleIntercept)/(data.RescaleSlope)
 return image
A_test, B_test, P_test, WEEK = {},{},{},{}
last_pat = ''
o1=[]
model.eval()
with torch.no_grad() :
t = tqdm(meta_df.Patient.values)
 for i,patient in enumerate(t):
meta_batch = torch.from_numpy(meta_df.values[i][1:-2][None].astype(np.float32)).cuda().float()
if patient != last_pat:
last_pat = patient
patient_dir = '.. /input/osic-pulmonary-fibrosis-progression/test/' + patient
imgs, meta, slices = load_scan(patient_dir)
 for i,img in enumerate(imgs):
imgs[i] = convertToHU(img, meta)
 imgs = crop(imgs)
 imgs = resize(imgs, shape=(50,512,512))
imgs = imgs[10:40]
imgs1,imgs2,imgs3 = np.zeros(( 30,512,512,1)) ,np.zeros(( 30,512,512,1)) ,np.zeros(( 30,512,512,1))
for i,img in enumerate(imgs):
imgs1[i] = window(img, -600, 1500)[:,:,None]
imgs2[i] = window(img, 100, 700)[:,:,None]
imgs3[i] = window(img, 40, 400)[:,:,None]
imgs_ = np.concatenate([imgs1, imgs2, imgs3], axis=-1)
 image_batch = []
for i in range(1):
imgs = imgs_[15]
imgs = np.rollaxis(imgs, -1, 0)/ 255.
 image_batch.append(imgs)
 break
image_batch = np.array(image_batch)
 p = p[0]
o=[]
for model in nets:
for img in image_batch:
img = img[None]
img = torch.from_numpy(img ).cuda().float()
o.append(model(img, meta_batch ).cpu().numpy())
 for o in np.mean(np.array(o), axis=0):
o1.append(o)
 o1=np.array(o1 )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
sub = data[data.WHERE=='test'].copy()
sub['FVC'] = o1[:, 1]
sub['Confidence'] = o1[:, 2] - o1[:, 0]
subm = sub[['Patient_Week','FVC','Confidence']].copy()
subm.head()
from fastai.vision import *
import fastai<define_variables>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
path = Path('.. /input' )<load_from_csv>
sub = subm.copy()
<load_from_csv><EOS> 
sub.to_csv("submission.csv", index=False )