Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<train_model>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
STEPS_PER_EPOCH =(count_data_items(train_filenames1)) // BATCH_SIZE
model.fit(train1, epochs=EPOCHS, callbacks=[lr_schedule],steps_per_epoch=STEPS_PER_EPOCH,validation_data=valid_dataset1,validation_steps=validationsize1//BATCH_SIZE )<load_from_csv>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
num_test_images = count_data_items(test_filenames1)
 submission_df1 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames1, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df1.copy()
del temp['target']
submission_df1['target'] += temp.merge(pred_df,on="image_name")['target']/TTA<save_to_csv>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
submission_df1.to_csv('efficientnetb7_784.csv', index = False )<split>
seed_everything(42 )
DATASET2 = 'melanoma-512x512'
GCS_PATH2 = KaggleDatasets().get_gcs_path(DATASET2)
 SIZE2 = [512,512]
train_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/train*.tfrec')
 test_filenames2 = tf.io.gfile.glob(GCS_PATH2 + '/test*.tfrec')
 train_filenames2, valid_filenames2 = train_test_split(train_filenames2, test_size = 0.15, random_state = SEED)
 def decode_image(image):
image = tf.image.decode_jpeg(image, channels = 3)
 image = tf.cast(image, tf.float32)/255.0
image = tf.reshape(image, [*SIZE2, 3])
 return image
def data_augment(image, label = None, seed = SEED):
image = tf.image.rot90(image, k = np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed = SEED)
 image = tf.image.random_flip_up_down(image, seed = SEED)
 if label is None:
return image
else:
return image, label
def read_labeled_tfrecord(example):
LFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'target': tf.io.FixedLenFeature([], tf.int64)}
example = tf.io.parse_single_example(example, LFormat)
 image = decode_image(example['image'])
 label = tf.cast(example['target'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UFormat = {'image': tf.io.FixedLenFeature([], tf.string),
'image_name': tf.io.FixedLenFeature([], tf.string)}
example = tf.io.parse_single_example(example, UFormat)
 image = decode_image(example['image'])
 image_name = example['image_name']
return image, image_name
def load_dataset(filenames, labeled = True, ordered = False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset =(tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO ).with_options(ignore_order ).map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO))
return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n)
 train2 =(load_dataset(train_filenames2 ).map(data_augment, num_parallel_calls = AUTO ).shuffle(SEED ).batch(BATCH_SIZE, drop_remainder = True ).repeat().prefetch(AUTO))
valid_dataset2 =(load_dataset(valid_filenames2, labeled=True ).batch(BATCH_SIZE ).repeat().prefetch(AUTO))<choose_model_class>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
with strategy.scope() :
model2 = tf.keras.Sequential([
efn.EfficientNetB7(input_shape=(*SIZE2, 3),weights='imagenet',pooling='avg',include_top=False),
Dense(1, activation='sigmoid')
 ])
 model2.compile(optimizer='adam',loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = LABEL_SMOOTHING),
metrics=[tf.keras.metrics.AUC(name='auc')] )<count_values>
Dropout_model = 0.3855
FVC_weight = 0.2
Confidence_weight = 0.2
validationsize2 = count_data_items(valid_filenames2)
 print(validationsize2 )<train_model>
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'] )
STEPS_PER_EPOCH =(count_data_items(train_filenames2)) // BATCH_SIZE
model2.fit(train2, epochs=EPOCHS, callbacks=[lr_schedule],steps_per_epoch=STEPS_PER_EPOCH,validation_data=valid_dataset2,validation_steps=validationsize2//BATCH_SIZE )<load_from_csv>
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111
num_test_images = count_data_items(test_filenames2)
 submission_df2 = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames2, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model2.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df2.copy()
del temp['target']
submission_df2['target'] += temp.merge(pred_df,on="image_name")['target']/TTA<save_to_csv>
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105
submission_df2.to_csv('efficientnetb7_512.csv', index=False )<init_hyperparams>
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158
fold_number = 1
seed = 66
debug = False
tta = 2 if debug else 20
batch_size = {
'tpu': 10,
'gpu': 22,
'cpu': 4,
}
arch = 'efficientnet-b5'
resolution = 456
input_res = 512
lr = 8e-6
weight_decay = 2e-5
pos_weight = 3.2
label_smoothing = 0.03
max_epochs = 7<set_options>
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107
%reload_ext autoreload
%autoreload 2
if 'TPU_NAME' in os.environ.keys() :
try:
except:
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev
else:
!pip install --pre torch==1.7.0.dev20200701+cu101 torchvision==0.8.0.dev20200701+cu101 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html
!pip install -U pip albumentations==0.4.5 PyYAML pytorch-lightning==0.8.5 efficientnet_pytorch<define_variables>
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125
num_workers = 2
gpus = 1 if torch.cuda.is_available() else None
try:
tpu_cores = 8
except:
tpu_cores = None
if isinstance(batch_size, dict):
if tpu_cores:
batch_size = batch_size['tpu']
lr *= tpu_cores
num_workers = 1
elif gpus:
batch_size = batch_size['gpu']
if 'K80' in torch.cuda.get_device_name() :
batch_size = batch_size//3
elif 'T4' in torch.cuda.get_device_name() :
batch_size = int(batch_size * 0.66)
 else:
batch_size = batch_size['cpu']
lr *= batch_size
dict(
 num_workers=num_workers,
tpu_cores=tpu_cores,
gpus=gpus,
batch_size=batch_size,
lr=lr,
)<install_modules>
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089
if gpus and not hasattr(torch.cuda, "amp"):
try:
except:
!git clone https://github.com/NVIDIA/apex nv_apex
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext"./nv_apex 
<set_options>
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100
warnings.filterwarnings("ignore")
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(seed*6 + fold_number)
 torch.__version__<normalization>
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096
class ImageDataset(Dataset):
def __init__(self, path, image_ids, labels=None, transforms=None):
super().__init__()
self.path = path
self.image_ids = image_ids
self.labels = labels
self.transforms = transforms
def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.path}/{image_id}.jpg', cv2.IMREAD_COLOR)
 if self.transforms:
sample = self.transforms(image=image)
 image = sample['image']
label = self.labels[idx] if self.labels is not None else 0.5
return image, label
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_labels(self):
return list(self.labels )<define_search_model>
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100
def get_train_transforms() :
return A.Compose([
A.JpegCompression(p=0.5),
A.Rotate(limit=80, p=1.0),
A.OneOf([
A.OpticalDistortion() ,
A.GridDistortion() ,
A.IAAPiecewiseAffine() ,
]),
A.RandomSizedCrop(min_max_height=(int(resolution*0.7), input_res),
height=resolution, width=resolution, p=1.0),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.GaussianBlur(p=0.3),
A.OneOf([
A.RandomBrightnessContrast() ,
A.HueSaturationValue() ,
]),
A.Cutout(num_holes=8, max_h_size=resolution//8, max_w_size=resolution//8, fill_value=0, p=0.3),
A.Normalize() ,
ToTensorV2() ,
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.CenterCrop(height=resolution, width=resolution, p=1.0),
A.Normalize() ,
ToTensorV2() ,
], p=1.0)
 def get_tta_transforms() :
return A.Compose([
A.JpegCompression(p=0.5),
A.RandomSizedCrop(min_max_height=(int(resolution*0.9), int(resolution*1.1)) ,
height=resolution, width=resolution, p=1.0),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Transpose(p=0.5),
A.Normalize() ,
ToTensorV2() ,
], p=1.0 )<load_from_csv>
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
DATA_PATH = '.. /input/melanoma-merged-external-data-512x512-jpeg'
TRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'
TEST_ROOT_PATH = f'{DATA_PATH}/512x512-test/512x512-test'
df_folds = pd.read_csv(f'{DATA_PATH}/folds.csv', index_col='image_id',
usecols=['image_id', 'fold', 'target'], dtype={'fold': np.byte, 'target': np.byte})
 _ = df_folds.groupby('fold' ).target.hist(alpha=0.4)
 df_folds.groupby('fold' ).target.mean().to_frame('ratio' ).T<load_from_csv>
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093
df_test = pd.read_csv(f'.. /input/siim-isic-melanoma-classification/test.csv', index_col='image_name')
 if debug:
df_folds = df_folds.sample(batch_size * 80)
 df_folds = df_folds.sample(frac=1.0, random_state=seed*6+fold_number )<create_dataframe>
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283
ds_train = ImageDataset(
 path=TRAIN_ROOT_PATH,
image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
labels=df_folds[df_folds['fold'] != fold_number].target.values,
transforms=get_train_transforms() , 
)
 ds_val = ImageDataset(
 path=TRAIN_ROOT_PATH,
image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
labels=df_folds[df_folds['fold'] == fold_number].target.values,
transforms=get_valid_transforms() , 
)
 ds_test = ImageDataset(
 path=TEST_ROOT_PATH,
image_ids=df_test.index.values,
transforms=get_tta_transforms() , 
)
 del df_folds
len(ds_train), len(ds_val), len(ds_test )<choose_model_class>
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087
class Model(pl.LightningModule):
def __init__(self, *args, **kwargs):
super().__init__()
self.net = EfficientNet.from_pretrained(arch, advprop=True)
 self.net._fc = nn.Linear(in_features=self.net._fc.in_features, out_features=1, bias=True)
 def forward(self, x):
return self.net(x)
 def configure_optimizers(self):
optimizer = torch.optim.AdamW(self.parameters() , lr=lr, weight_decay=weight_decay)
 scheduler = torch.optim.lr_scheduler.OneCycleLR(
 max_lr=lr,
epochs=max_epochs,
optimizer=optimizer,
steps_per_epoch=int(len(ds_train)/ batch_size),
pct_start=0.1,
div_factor=10,
final_div_factor=100,
base_momentum=0.90,
max_momentum=0.95, 
)
 return [optimizer], [scheduler]
def step(self, batch):
x, y = batch
y_hat = self(x ).flatten()
y_smo = y.float() *(1 - label_smoothing)+ 0.5 * label_smoothing
loss = F.binary_cross_entropy_with_logits(y_hat, y_smo.type_as(y_hat),
pos_weight=torch.tensor(pos_weight))
return loss, y, y_hat.sigmoid()
def training_step(self, batch, batch_nb):
loss, y, y_hat = self.step(batch)
 acc =(y_hat.round() == y ).float().mean().item()
tensorboard_logs = {'train_loss': loss, 'acc': acc}
return {'loss': loss, 'acc': acc, 'log': tensorboard_logs}
def validation_step(self, batch, batch_nb):
loss, y, y_hat = self.step(batch)
 return {'val_loss': loss,
'y': y.detach() , 'y_hat': y_hat.detach() }
def validation_epoch_end(self, outputs):
avg_loss = torch.stack([x['val_loss'] for x in outputs] ).mean()
y = torch.cat([x['y'] for x in outputs])
 y_hat = torch.cat([x['y_hat'] for x in outputs])
 auc = AUROC()(pred=y_hat, target=y)if y.float().mean() > 0 else 0.5
acc =(y_hat.round() == y ).float().mean().item()
print(f"Epoch {self.current_epoch} acc:{acc} auc:{auc}")
 tensorboard_logs = {'val_loss': avg_loss, 'val_auc': auc, 'val_acc': acc}
return {'avg_val_loss': avg_loss,
'val_auc': auc, 'val_acc': acc,
'log': tensorboard_logs}
def test_step(self, batch, batch_nb):
x, _ = batch
y_hat = self(x ).flatten().sigmoid()
return {'y_hat': y_hat}
def test_epoch_end(self, outputs):
y_hat = torch.cat([x['y_hat'] for x in outputs])
 df_test['target'] = y_hat.tolist()
N = len(glob('submission*.csv'))
df_test.target.to_csv(f'submission{N}.csv')
 return {'tta': N}
def train_dataloader(self):
return DataLoader(ds_train, batch_size=batch_size, num_workers=num_workers,
drop_last=True, shuffle=True, pin_memory=True)
 def val_dataloader(self):
return DataLoader(ds_val, batch_size=batch_size, num_workers=num_workers,
drop_last=False, shuffle=False, pin_memory=True)
 def test_dataloader(self):
return DataLoader(ds_test, batch_size=batch_size, num_workers=num_workers,
drop_last=False, shuffle=False, pin_memory=False)
 model = Model()<train_model>
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090
del batch; del targets<set_options>
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
<choose_model_class>
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093
checkpoint_callback = pl.callbacks.ModelCheckpoint("{epoch:02d}_{val_auc:.4f}",
save_top_k=1, monitor='val_auc', mode='max')
 trainer = pl.Trainer(
 tpu_cores=tpu_cores,
gpus=gpus,
precision=16 if gpus else 32,
max_epochs=max_epochs,
num_sanity_val_steps=1 if debug else 0,
checkpoint_callback=checkpoint_callback,
)<set_options>
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402
torch.cuda.empty_cache() ; gc.collect()
torch.cuda.empty_cache() ; gc.collect()<train_model>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1
trainer.fit(model )<save_to_csv>
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0
submission = df_test[['target']]
submission.target = 0.0
for sub in glob('submission*.csv'):
submission.target += pd.read_csv(sub, index_col='image_name' ).target
submission.target -= submission.target.min()
submission.target /= submission.target.max()
submission.to_csv(f'submission_fold{fold_number}.csv')
 submission.hist(bins=100, log=True, alpha=0.6)
 submission.target.describe()<load_from_csv>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
folds_path = '.. /input/melanoma-neat-pytorch-lightning'
!cp {folds_path}/*_fold*.csv.
 !cp {folds_path}/*.ckpt .<save_to_csv>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
folds_sub = pd.read_csv(f'{folds_path}/submission.csv', index_col='image_name')
 submission.target += folds_sub.target *(fold_number + 4)
 submission.target /=(fold_number + 5)
 submission.to_csv('submission.csv')
 submission.hist(bins=100, log=True, alpha=0.6)
 submission.target.describe()<set_options>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
if not debug and gpus:
!rm nv_apex -rf
!ls -sh<set_options>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
warnings.filterwarnings('ignore' )<load_from_csv>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
df_train = pd.read_csv("/kaggle/input/santander-customer-transaction-prediction-dataset/train.csv")
 df_test = pd.read_csv("/kaggle/input/santander-customer-transaction-prediction-dataset/test.csv" )<load_from_csv>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )
sample_submission = pd.read_csv("/kaggle/input/santander-customer-transaction-prediction/sample_submission.csv" )<count_missing_values>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
df_train.isnull().sum().sum()<count_missing_values>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
df_test.isnull().sum().sum()<count_values>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
df_train.target.value_counts(normalize=True )<groupby>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
df_train.loc[df_train.target == 0][feat].mean(axis=1 )<groupby>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
df_train.loc[df_train.target == 1][feat].mean(axis=1 )<groupby>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
df_train.loc[df_train.target == 0][feat].mean()<groupby>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
df_train.loc[df_train.target == 1][feat].mean()<import_modules>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
import lightgbm as lgb
from sklearn.metrics import f1_score, classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import StratifiedKFold<drop_column>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
X= df_train.drop(["target","ID_code"],axis=1)
 X.head()<prepare_x_and_y>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
Y=df_train.target
Y.head()<drop_column>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
X_test = df_test.drop("ID_code",axis=1)
 X_test.head()<init_hyperparams>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
params = {'objective' : "binary",
'boost':"gbdt",
'metric':"auc",
'boost_from_average':"false",
'num_threads':8,
'learning_rate' : 0.01,
'num_leaves' : 13,
'max_depth':-1,
'tree_learner' : "serial",
'feature_fraction' : 0.05,
'bagging_freq' : 5,
'bagging_fraction' : 0.4,
'min_data_in_leaf' : 80,
'min_sum_hessian_in_leaf' : 10.0,
'verbosity' : 1}<split>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
num_folds = 10
folds = StratifiedKFold(n_splits=num_folds, shuffle=False, random_state=42)
 oof = np.zeros(len(df_train))
y_pred_lgbm = np.zeros(len(df_train.target))
print('Light GBM Model')
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train.values,df_train.target.values)) :
print('Fold', fold_)
 X_train, Y_train = df_train.iloc[trn_idx][feat], df_train.target.iloc[trn_idx]
X_valid, Y_valid = df_train.iloc[val_idx][feat], df_train.target.iloc[val_idx]
train_data = lgb.Dataset(X_train, label=Y_train)
 valid_data = lgb.Dataset(X_valid, label=Y_valid)
 lgb_model = lgb.train(params, train_data, 1000000, valid_sets = [train_data, valid_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = lgb_model.predict(df_train.iloc[val_idx][feat], num_iteration=lgb_model.best_iteration)
 y_pred_lgbm += lgb_model.predict(df_test[feat], num_iteration=lgb_model.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(df_train.target, oof)) )<import_modules>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
from sklearn.model_selection import train_test_split<import_modules>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
from sklearn.model_selection import train_test_split<split>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
train_X, val_X, train_y, val_y = train_test_split(X, Y, random_state=42)
 rfc_mod = RandomForestClassifier(random_state=42 ).fit(train_X, train_y )<predict_on_test>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
y_pred_rfc = rfc_mod.predict(X_test )<compute_test_metric>
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print("F1 Score :",f1_score(y_pred_rfc,Y,average = "weighted"))
print('Report:
',classification_report(Y, y_pred_rfc))
print('Confusion Matrix:
',confusion_matrix(Y, y_pred_rfc))<split>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
train_X, val_X, train_y, val_y = train_test_split(X, Y, random_state=42)
 dectre_mod = DecisionTreeClassifier(random_state=42, max_depth=5, min_samples_split=5 ).fit(train_X, train_y )<predict_on_test>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
y_pred_dec_tree = dectre_mod.predict(X_test )<compute_test_metric>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
print("F1 Score :",f1_score(y_pred_dec_tree,Y,average = "weighted"))
print('Report:
',classification_report(Y, y_pred_dec_tree))
print('Confusion Matrix:
',confusion_matrix(Y, y_pred_dec_tree))<split>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
train_X, val_X, train_y, val_y = train_test_split(X,Y,random_state=42)
 logreg_mod = LogisticRegression(random_state=42 ).fit(train_X, train_y )<predict_on_test>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
y_pred_logreg = logreg_mod.predict(X_test )<compute_test_metric>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
print("F1 Score :",f1_score(y_pred_logreg,Y,average = "weighted"))
print('Report:
',classification_report(Y, y_pred_logreg))
print('Confusion Matrix:
',confusion_matrix(Y, y_pred_logreg))<save_to_csv>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
submission_lgbm = pd.DataFrame({
"ID_code": df_test["ID_code"],
"target": y_pred_lgbm
})
 submission_lgbm.to_csv('submission_lgbm.csv', index=False )<save_to_csv>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
<save_to_csv><EOS> 
df.to_csv('submission.csv', index=False )