Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<compute_train_metric>
import numpy as np
import pandas as pd
import pydicom
import os
import random
import matplotlib.pyplot as plt
from tqdm import tqdm
from PIL import Image
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold
import cv2
import time
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
predict_valid_1,label_valid_1=Run_validation()
preds_valid_1=optR.predict(predict_valid_1,coefficients)
 confusionMatrics(preds_valid_1,label_valid_1 )<compute_test_metric>
def seed_everything(seed=1000000007):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 print(time.time())
 seed_everything(int(time.time()))
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
All_data = tr.append([chunk, sub] )
print("Train Cohen Kappa score: %.3f" % cohen_kappa_score(preds_valid_1, label_valid_1, weights='quadratic'))<compute_train_metric>
All_data['min_week'] = All_data['Weeks']
All_data.loc[All_data.WHERE=='test','min_week'] = np.nan
All_data['min_week'] = All_data.groupby('Patient')['min_week'].transform('min' )
coefficients_1=[0.5, 1.5, 2.5, 3.5]
preds_valid_2=optR.predict(predict_valid_1,coefficients_1)
 confusionMatrics(preds_valid_2,label_valid_1 )<compute_test_metric>
base = All_data.loc[All_data.Weeks == All_data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
print("Train Cohen Kappa score: %.3f" % cohen_kappa_score(preds_valid_2, label_valid_1, weights='quadratic'))<predict_on_test>
All_data = All_data.merge(base, on='Patient', how='left')
 All_data['base_week'] = All_data['Weeks'] - All_data['min_week']
for i in tqdm(range(10)) :
test_preds=[]
torch.cuda.empty_cache()
test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)
 test_preds1 = np.zeros(( len(test_dataset), 1))
tk0 = tqdm(test_data_loader)
 for i, x_batch in enumerate(tk0):
with torch.no_grad() :
x_batch = x_batch["image"]
pred = model(x_batch.to(device))
test_preds1[i * batch_size:(i + 1)* batch_size] = pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)
 test_preds.append(test_preds1)
 predict = np.mean(test_preds, axis=0 )<save_to_csv>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in All_data[col].unique() :
FE.append(mod)
 All_data[mod] =(All_data[col] == mod ).astype(int)
test_preds=optR.predict(predict,coefficients)
 sample = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 sample.diagnosis = test_preds.astype(int)
 sample.to_csv("submission.csv", index=False )<save_model>
def csv_normlize(data,attr1,attr):
data[attr1] =(data[attr] - data[attr].min())/(data[attr].max() - data[attr].min())
 csv_normlize(All_data,'age','Age')
 csv_normlize(All_data,'BASE','min_FVC')
 csv_normlize(All_data,'week','base_week')
 csv_normlize(All_data,'percent','Percent')
 FE += ['age','percent','week','BASE']
torch.save(model,"final_traindv25.pth" )<feature_engineering>
root_dir = '/kaggle/input/osic-pulmonary-fibrosis-progression/train'
test_dir = '/kaggle/input/osic-pulmonary-fibrosis-progression/test'
desk_dir = 'save_output'
model_file = '/kaggle/working/diophantus.pt'
resize_dims =(40, 256, 256)
 clip_bounds =(-1000, 200)
 watershed_iterations = 1
pre_calculated_mean = 0.02865046213070556
latent_features = 3
batch_size = 16
learning_rate = 1e-4
num_epochs = 3
val_size = 0.2
tensorboard_dir = '/kaggle/working/runs'
start = time.time()<set_options>
class CTScansDataset(Dataset):
def __init__(self, root_dir, transform=None):
self.root_dir = Path(root_dir)
 self.patients = [p for p in self.root_dir.glob('*')if p.is_dir() ]
self.transform = transform
def __len__(self):
return len(self.patients)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
image, metadata = self.load_scan(self.patients[idx])
 sample = {'image': image, 'metadata': metadata}
if self.transform:
sample = self.transform(sample)
 return sample
def save(self, path):
t0 = time()
Path(path ).mkdir(exist_ok=True, parents=True)
 print('Saving pre-processed dataset to disk')
 sleep(1)
 cum = 0
bar = trange(len(self))
for i in bar:
try:
sample = self[i]
image, data = sample['image'], sample['metadata']
cum += torch.mean(image ).item()
bar.set_description(f'Saving CT scan {data.PatientID}')
 fname = Path(path)/ f'{data.PatientID}.pt'
torch.save(image, fname)
 except:
print(i)
 pass
sleep(1)
 bar.close()
print(f'Done! Time {timedelta(seconds=time() - t0)}
'
f'Tot value: {cum }')
 def get_patient(self, patient_id):
patient_ids = [str(p.stem)for p in self.patients]
return self.__getitem__(patient_ids.index(patient_id))
@staticmethod
def load_scan(path):
slices = [pydicom.read_file(p)for p in path.glob('*.dcm')]
try:
slices.sort(key=lambda x: float(x.ImagePositionPatient[2]))
except AttributeError:
warnings.warn(f'Patient {slices[0].PatientID} CT scan does not '
f'have "ImagePositionPatient".Assuming filenames '
f'in the right scan order.')
 image = np.stack([s.pixel_array.astype(float)for s in slices])
 return image, slices[0]
class CropBoundingBox:
@staticmethod
def bounding_box(img3d: np.array):
mid_img = img3d[int(img3d.shape[0] / 2)]
same_first_row =(mid_img[0, :] == mid_img[0, 0] ).all()
same_first_col =(mid_img[:, 0] == mid_img[0, 0] ).all()
if same_first_col and same_first_row:
return True
else:
return False
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
if not self.bounding_box(image):
return sample
mid_img = image[int(image.shape[0] / 2)]
r_min, r_max = None, None
c_min, c_max = None, None
for row in range(mid_img.shape[0]):
if not(mid_img[row, :] == mid_img[0, 0] ).all() and r_min is None:
r_min = row
if(mid_img[row, :] == mid_img[0, 0] ).all() and r_max is None \
and r_min is not None:
r_max = row
break
for col in range(mid_img.shape[1]):
if not(mid_img[:, col] == mid_img[0, 0] ).all() and c_min is None:
c_min = col
if(mid_img[:, col] == mid_img[0, 0] ).all() and c_max is None \
and c_min is not None:
c_max = col
break
image = image[:, r_min:r_max, c_min:c_max]
return {'image': image, 'metadata': data}
class ConvertToHU:
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
img_type = data.ImageType
is_hu = img_type[0] == 'ORIGINAL' and not(img_type[2] == 'LOCALIZER')
 intercept = data.RescaleIntercept
slope = data.RescaleSlope
image =(image * slope + intercept ).astype(np.int16)
 return {'image': image, 'metadata': data}
class Resize:
def __init__(self, output_size):
assert isinstance(output_size, tuple)
 self.output_size = output_size
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
resize_factor = np.array(self.output_size)/ np.array(image.shape)
 image = zoom(image, resize_factor, mode='nearest')
 return {'image': image, 'metadata': data}

 class Clip:
def __init__(self, bounds=(-1000, 500)) :
self.min = min(bounds)
 self.max = max(bounds)
 def __call__(self, sample):
image, data = sample['image'], sample['metadata']
image[image < self.min] = self.min
image[image > self.max] = self.max
return {'image': image, 'metadata': data}
class MaskWatershed:
def __init__(self, min_hu, iterations, show_tqdm):
self.min_hu = min_hu
self.iterations = iterations
self.show_tqdm = show_tqdm
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
stack = []
if self.show_tqdm:
bar = trange(image.shape[0])
 bar.set_description(f'Masking CT scan {data.PatientID}')
 else:
bar = range(image.shape[0])
 for slice_idx in bar:
sliced = image[slice_idx]
stack.append(self.seperate_lungs(sliced, self.min_hu,
self.iterations))
return {
'image': np.stack(stack),
'metadata': sample['metadata']
}
@staticmethod
def seperate_lungs(image, min_hu, iterations):
h, w = image.shape[0], image.shape[1]
marker_internal, marker_external, marker_watershed = MaskWatershed.generate_markers(image)
 sobel_filtered_dx = ndimage.sobel(image, 1)
 sobel_filtered_dy = ndimage.sobel(image, 0)
 sobel_gradient = np.hypot(sobel_filtered_dx, sobel_filtered_dy)
 sobel_gradient *= 255.0 / np.max(sobel_gradient)
 watershed = morphology.watershed(sobel_gradient, marker_watershed)
 outline = ndimage.morphological_gradient(watershed, size=(3,3))
outline = outline.astype(bool)
 blackhat_struct = [[0, 0, 1, 1, 1, 0, 0],
[0, 1, 1, 1, 1, 1, 0],
[1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1, 1],
[0, 1, 1, 1, 1, 1, 0],
[0, 0, 1, 1, 1, 0, 0]]
blackhat_struct = ndimage.iterate_structure(blackhat_struct, iterations)
 outline += ndimage.black_tophat(outline, structure=blackhat_struct)
 lungfilter = np.bitwise_or(marker_internal, outline)
 lungfilter = ndimage.morphology.binary_closing(lungfilter, structure=np.ones(( 5,5)) , iterations=3)
 segmented = np.where(lungfilter == 1, image, min_hu * np.ones(( h, w)))
 return segmented
@staticmethod
def generate_markers(image, threshold=-400):
h, w = image.shape[0], image.shape[1]
marker_internal = image < threshold
marker_internal = segmentation.clear_border(marker_internal)
 marker_internal_labels = measure.label(marker_internal)
 areas = [r.area for r in measure.regionprops(marker_internal_labels)]
areas.sort()
if len(areas)> 2:
for region in measure.regionprops(marker_internal_labels):
if region.area < areas[-2]:
for coordinates in region.coords:
marker_internal_labels[coordinates[0], coordinates[1]] = 0
marker_internal = marker_internal_labels > 0
external_a = ndimage.binary_dilation(marker_internal, iterations=10)
 external_b = ndimage.binary_dilation(marker_internal, iterations=55)
 marker_external = external_b ^ external_a
marker_watershed = np.zeros(( h, w), dtype=np.int)
 marker_watershed += marker_internal * 255
marker_watershed += marker_external * 128
return marker_internal, marker_external, marker_watershed
class Normalize:
def __init__(self, bounds=(-1000, 500)) :
self.min = min(bounds)
 self.max = max(bounds)
 def __call__(self, sample):
image, data = sample['image'], sample['metadata']
image = image.astype(np.float)
 image =(image - self.min)/(self.max - self.min)
 return {'image': image, 'metadata': data}
class ToTensor:
def __init__(self, add_channel=True):
self.add_channel = add_channel
def __call__(self, sample):
image, data = sample['image'], sample['metadata']
if self.add_channel:
image = np.expand_dims(image, axis=0)
 return {'image': torch.from_numpy(image), 'metadata': data}
class ZeroCenter:
def __init__(self, pre_calculated_mean):
self.pre_calculated_mean = pre_calculated_mean
def __call__(self, tensor):
return tensor - self.pre_calculated_mean
%pylab inline 
<import_modules>
test = CTScansDataset(
 root_dir=test_dir,
transform=transforms.Compose([
CropBoundingBox() ,
ConvertToHU() ,
Resize(resize_dims),
Clip(bounds=clip_bounds),
MaskWatershed(
 min_hu=min(clip_bounds),
iterations=watershed_iterations,
show_tqdm=False),
Normalize(bounds=clip_bounds),
ToTensor()
]))
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedShuffleSplit<import_modules>
class CTTensorsDataset(Dataset):
def __init__(self, root_dir, transform=None):
self.root_dir = Path(root_dir)
 self.tensor_files = sorted([f for f in self.root_dir.glob('*.pt')])
 self.transform = transform
def __len__(self):
return len(self.tensor_files)
 def __getitem__(self, item):
if torch.is_tensor(item):
item = item.tolist()
image = torch.load(self.tensor_files[item])
 if self.transform:
image = self.transform(image)
 return {
'patient_id': self.tensor_files[item].stem,
'image': image
}
def mean(self):
cum = 0
for i in range(len(self)) :
sample = self[i]['image']
cum += torch.mean(sample ).item()
return cum / len(self)
 def random_split(self, val_size: float):
num_val = int(val_size * len(self))
num_train = len(self)- num_val
return random_split(self, [num_train, num_val] )
from keras.models import Sequential
from keras.layers import merge
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers.advanced_activations import PReLU
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils.np_utils import to_categorical
from keras.callbacks import EarlyStopping<load_from_csv>
train = CTTensorsDataset(
 root_dir='.. /input/osic-cached-dataset',
transform=ZeroCenter(pre_calculated_mean=pre_calculated_mean)
)
 cum = 0
for i in range(len(train)) :
sample = train[i]['image']
cum += torch.mean(sample ).item()
assert cum / len(train)== pytest.approx(0 )
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )<categorify>
class AutoEncoder(nn.Module):
def __init__(self, latent_features=latent_features):
super(AutoEncoder, self ).__init__()
self.conv1 = nn.Conv3d(1, 16, 3)
 self.conv2 = nn.Conv3d(16, 32, 3)
 self.conv3 = nn.Conv3d(32, 96, 2)
 self.conv4 = nn.Conv3d(96, 1, 1)
 self.pool1 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True)
 self.pool2 = nn.MaxPool3d(kernel_size=3, stride=3, return_indices=True)
 self.pool3 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True)
 self.pool4 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True)
 self.fc1 = nn.Linear(10 * 10, latent_features)
 self.fc2 = nn.Linear(latent_features, 10 * 10)
 self.deconv0 = nn.ConvTranspose3d(1, 96, 1)
 self.deconv1 = nn.ConvTranspose3d(96, 32, 2)
 self.deconv2 = nn.ConvTranspose3d(32, 16, 3)
 self.deconv3 = nn.ConvTranspose3d(16, 1, 3)
 self.unpool0 = nn.MaxUnpool3d(kernel_size=2, stride=2)
 self.unpool1 = nn.MaxUnpool3d(kernel_size=2, stride=2)
 self.unpool2 = nn.MaxUnpool3d(kernel_size=3, stride=3)
 self.unpool3 = nn.MaxUnpool3d(kernel_size=2, stride=2)
 def encode(self, x, return_partials=True):
x = self.conv1(x)
 up3out_shape = x.shape
x, i1 = self.pool1(x)
 x = self.conv2(x)
 up2out_shape = x.shape
x, i2 = self.pool2(x)
 x = self.conv3(x)
 up1out_shape = x.shape
x, i3 = self.pool3(x)
 x = self.conv4(x)
 up0out_shape = x.shape
x, i4 = self.pool4(x)
 x = x.view(-1, 10 * 10)
 x = F.relu(self.fc1(x))
if return_partials:
return x, up3out_shape, i1, up2out_shape, i2, up1out_shape, i3, \
up0out_shape, i4
else:
return x
def forward(self, x):
x, up3out_shape, i1, up2out_shape, i2, \
up1out_shape, i3, up0out_shape, i4 = self.encode(x)
 x = F.relu(self.fc2(x))
x = x.view(-1, 1, 1, 10, 10)
 x = self.unpool0(x, output_size=up0out_shape, indices=i4)
 x = self.deconv0(x)
 x = self.unpool1(x, output_size=up1out_shape, indices=i3)
 x = self.deconv1(x)
 x = self.unpool2(x, output_size=up2out_shape, indices=i2)
 x = self.deconv2(x)
 x = self.unpool3(x, output_size=up3out_shape, indices=i1)
 x = self.deconv3(x)
 return x
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape )<normalization>
t0 = time()
data = CTTensorsDataset(
 root_dir='.. /input/osic-cached-dataset',
transform=ZeroCenter(pre_calculated_mean=pre_calculated_mean)
)
 train_set, val_set = data.random_split(val_size)
 datasets = {'train': train_set, 'val': val_set}
dataloaders = {
x: DataLoader(
 datasets[x],
batch_size=batch_size,
shuffle=(x == 'train'),
num_workers=2 
)for x in ['train', 'val']}
dataset_sizes = {x: len(datasets[x])for x in ['train', 'val']}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model = AutoEncoder(latent_features=latent_features ).to(device)
 criterion = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate)
 best_model_wts = None
best_loss = np.inf
date_time = datetime.now().strftime("%Y%m%d-%H%M")
 log_dir = Path(tensorboard_dir)/ f'{date_time}'
writer = SummaryWriter(log_dir )
X = preprocessing.MinMaxScaler().fit(data ).transform(data)
 X = StandardScaler().fit(data ).transform(data)
 print(X.shape)
 X<categorify>
latent_Dicom={}
for i in range(len(test)) :
Pid = test[i]['metadata'].PatientID
print(f'Inspecting CT Scan {Pid}')
 sample = test[i]['image'].squeeze(0 ).numpy()
with torch.no_grad() :
img = test[i]['image'].unsqueeze(0 ).float().to(device)
 latent_features = model.encode(img, return_partials=False)\ 
.squeeze().cpu().numpy().tolist()
latent_Dicom[Pid]=latent_features
y_cat = to_categorical(y)
 print(y_cat.shape )<split>
for i in range(len(data)) :
Pid=data[i]["patient_id"]
sample = data[i]['image'].squeeze(0 ).numpy()
with torch.no_grad() :
img = data[i]['image'].unsqueeze(0 ).float().to(device)
 latent_features = model.encode(img, return_partials=False)\ 
.squeeze().cpu().numpy().tolist()
latent_Dicom[Pid]=latent_features
sss = StratifiedShuffleSplit(n_splits=10, test_size=0.2,random_state=12345)
 train_index, val_index = next(iter(sss.split(X, y)))
 x_train, x_val = X[train_index], X[val_index]
y_train, y_val = y_cat[train_index], y_cat[val_index]
print("x_train dim: ",x_train.shape)
 print("x_val dim: ",x_val.shape )<choose_model_class>
latent_features_tot=3
model = Sequential()
model.add(Dense(768,input_dim=192, kernel_initializer='glorot_normal', activation='tanh'))
model.add(Dropout(0.4))
model.add(Dense(768, activation='tanh'))
model.add(Dropout(0.4))
model.add(Dense(99, activation='softmax'))<choose_model_class>
pl = All_data['Patient'].tolist()
pl2=[latent_Dicom[i] if i in latent_Dicom else np.nan for i in pl]
pl3=np.array(pl2)
 for i in range(latent_features_tot):
stri = 'latent_features'+str(i)
 pl4=pl3[:,i]
All_data[stri]=pl4
FE.append(stri )
model.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"] )<train_model>
tr = All_data.loc[All_data.WHERE=='train']
chunk = All_data.loc[All_data.WHERE=='val']
sub = All_data.loc[All_data.WHERE=='test']
tr.shape, chunk.shape, sub.shape
early_stopping = EarlyStopping(monitor='val_loss', patience=300)
 history = model.fit(x_train, y_train,batch_size=192,epochs=2500 ,verbose=0,
validation_data=(x_val, y_val),callbacks=[early_stopping] )<train_model>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print()
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))<load_from_csv>
y = tr['FVC'].values
z = tr[FE].values
ze = sub[FE].values
nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = preprocessing.MinMaxScaler().fit(test ).transform(test)
 test = StandardScaler().fit(test ).transform(test)
 yPred = model.predict_proba(test )<create_dataframe>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))<save_to_csv>
def score2(y_true, y_pred):
sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = np.abs(y_true - fvc_pred)
 delta[delta>1000]=1000
sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return np.mean(metric)
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv())
 end = time.time()
print(round(( end-start),2), "seconds" )<set_options>
%%time
cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
%pylab inline 
<import_modules>
print(score2(y,pred))
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder<import_modules>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation
from keras.utils.np_utils import to_categorical<set_options>
sub2=sub.copy(deep=True )
rcParams['figure.figsize'] = 10,10<load_from_csv>
sub2['FVC'] = pe[:, 1]
sub2['Confidence'] = pe[:, 2] - pe[:, 0]
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id' )<categorify>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
sub2.loc[sub2['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
sub2.loc[sub2['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<normalization><EOS> 
sub2[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )