Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<define_variables>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[str(i)]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )<concatenate>
%matplotlib inline
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42)
 config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)
 palette_ro = ["
ROOT = ".. /input/osic-pulmonary-fibrosis-progression/"
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape<normalization>
train = pd.read_csv(ROOT + "train.csv")
 test = pd.read_csv(ROOT + "test.csv")
 sub = pd.read_csv(ROOT + "sample_submission.csv")
 print("Training data shape: ", train.shape)
 print("Test data shape: ", test.shape)
 train.head(10 )
signals = X_test_input.reshape(( len(X_test_input), X_test_input.shape[1]*X_test_input.shape[2]))
features_test = []
for signal in signals:
features_test.append(entropy_and_fractal_dim(signal))<normalization>
train.isnull().sum()
features_test = np.array(features_test ).reshape(( len(features_test), 18))
features_test = scaler.transform(features_test)
 np.save("features_test.npy",features_test )<load_from_csv>
train.isnull().sum()
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
test.isnull().sum()
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict([X_test_input, features_test], batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )<data_type_conversions>
test.isnull().sum()
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
dupRows_train = train[train.duplicated(subset=['Patient', 'Weeks'], keep=False)]
print("There are {} duplicate rows here({:.2f} percent of the total ).".format(len(dupRows_train), len(dupRows_train)/len(train)*100))
dupRows_train
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
train.drop_duplicates(subset=['Patient', 'Weeks'], keep=False, inplace=True )
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import Model
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold,RepeatedStratifiedKFold
from keras.callbacks import *
import tensorflow as tf<define_variables>
stats = []
for col in train.columns:
stats.append(( col,
train[col].nunique() ,
train[col].value_counts().index[0],
train[col].value_counts().values[0],
train[col].isnull().sum() * 100 / train.shape[0],
train[col].value_counts(normalize=True, dropna=False ).values[0] * 100,
train[col].dtype))
stats_df = pd.DataFrame(stats, columns=['Feature', 'Unique values', 'Most frequent item', 'Freuquence of most frequent item', 'Percentage of missing values', 'Percentage of values in the biggest category', 'Type'])
 stats_df.sort_values('Percentage of missing values', ascending=False )
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
data = train.groupby("Patient" ).first().reset_index(drop=True)
 data.head()
def matthews_correlation(y_true, y_pred):

 y_pred = tf.convert_to_tensor(y_pred, np.float32)
 y_true = tf.convert_to_tensor(y_true, np.float32)
 y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
 tn = K.sum(y_neg * y_pred_neg)
 fp = K.sum(y_neg * y_pred_pos)
 fn = K.sum(y_pos * y_pred_neg)
 numerator =(tp * tn - fp * fn)
 denominator = K.sqrt(( tp + fp)*(tp + fn)*(tn + fp)*(tn + fn))
return numerator /(denominator + K.epsilon() )<choose_model_class>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<load_from_csv>
%%time
def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(528, 528, 1), model_class=None):
inp = L.Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = L.GlobalAveragePooling2D()(x)
 inp2 = L.Input(shape=(4,))
x2 = L.GaussianNoise(0.2 )(inp2)
 x = L.Concatenate()([x, x2])
 x = L.Dropout(0.32 )(x)
 x = L.Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ["b6"]
models = [build_model(model_class=m, shape=(528, 528, 1)) for m in model_classes]
print('Number of models: ' + str(len(models)) )
df_train = pd.read_csv('.. /input/metadata_train.csv')
 df_train = df_train.set_index(['id_measurement', 'phase'])
 df_train.head()<define_variables>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip * sq2)
 return np.mean(metric )
max_num = 127
min_num = -128<categorify>
tr_p, vl_p = train_test_split(P,
shuffle=True,
train_size=0.8)
 subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test, W, FVC = {}, {}, {}, {}, {}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, "FVC"] = fvc
sub.loc[sub.Patient_Week == k, "Confidence"] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
def min_max_transformation(ts, min_data, max_data, range_needed=(-1,1)) :
if min_data < 0:
ts_std =(ts + abs(min_data)) /(max_data + abs(min_data))
else:
ts_std =(ts - min_data)/(max_data - min_data)
 if range_needed[0] < 0:
return ts_std *(range_needed[1] + abs(range_needed[0])) + range_needed[0]
else:
return ts_std *(range_needed[1] - range_needed[0])+ range_needed[0]<categorify>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
def transform_ts(ts, n_dim=160, min_max=(-1,1)) :
ts_std = min_max_transformation(ts, min_data=min_num, max_data=max_num)
 bucket_size = int(sample_size / n_dim)
 new_ts = []
for i in range(0, sample_size, bucket_size):
ts_range = ts_std[i:i + bucket_size]
mean = ts_range.mean()
std = ts_range.std()
std_top = mean + std
std_bot = mean - std
percentil_calc = np.percentile(ts_range, [0, 1, 25, 50, 75, 99, 100])
 max_range = percentil_calc[-1] - percentil_calc[0]
relative_percentile = percentil_calc - mean
new_ts.append(np.concatenate([np.asarray([mean, std, std_top, std_bot, max_range]),percentil_calc, relative_percentile]))
return np.asarray(new_ts )<prepare_x_and_y>
sub[["Patient_Week", "FVC", "Confidence"]].to_csv("submission_img.csv", index=False )
def prepare_data(start, end):
praq_train = pq.read_pandas('.. /input/train.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
X = []
y = []
for id_measurement in tqdm(df_train.index.levels[0].unique() [int(start/3):int(end/3)]):
X_signal = []
for phase in [0,1,2]:
signal_id, target = df_train.loc[id_measurement].loc[phase]
if phase == 0:
y.append(target)
 X_signal.append(transform_ts(praq_train[str(signal_id)]))
X_signal = np.concatenate(X_signal, axis=1)
 X.append(X_signal)
 X = np.asarray(X)
 y = np.asarray(y)
 return X, y<concatenate>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
X = []
y = []
def load_all() :
total_size = len(df_train)
 for ini, end in [(0, int(total_size/2)) ,(int(total_size/2), total_size)]:
X_temp, y_temp = prepare_data(ini, end)
 X.append(X_temp)
 y.append(y_temp)
 load_all()
X = np.concatenate(X)
 y = np.concatenate(y )<train_model>
sub = pd.read_csv(ROOT + "sample_submission.csv")
 sub.head()
print(X.shape, y.shape)
 np.save("X.npy",X)
 np.save("y.npy",y )<create_dataframe>
sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient', 'Weeks', 'Confidence', 'Patient_Week']]
sub = sub.merge(test.drop('Weeks', axis=1), on="Patient")
 sub.head()
df_y = pd.DataFrame(y.reshape(1,-1))<normalization>
train['WHERE'] = 'train'
test['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = train.append([test, sub])
 print(train.shape, test.shape, sub.shape, data.shape)
 print(train.Patient.nunique() , test.Patient.nunique() , sub.Patient.nunique() , data.Patient.nunique())
 data.head(10 )
tsne = TSNE(n_components=2, random_state=42, verbose = 2)
 TSNE_X = tsne.fit_transform(df )<normalization>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test', 'min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 data.head(10 )
isomap = Isomap(n_components=200)
 df_reduced = isomap.fit_transform(df )<set_options>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient', 'FVC']].copy()
base.columns = ['Patient', 'base_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 base.head()
lin_pca = KernelPCA(n_components = 2, kernel="linear", fit_inverse_transform=True)
 rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.0433, fit_inverse_transform=True)
 sig_pca = KernelPCA(n_components = 2, kernel="sigmoid", gamma=0.001, coef0=1, fit_inverse_transform=True)
 plt.figure(figsize=(11, 4))
for subplot, pca, title in(( 131, lin_pca, "Linear kernel"),(132, rbf_pca, "RBF kernel, $\gamma=0.04$"),(133, sig_pca, "Sigmoid kernel, $\gamma=10^{-3}, r=1$")) :
X_reduced = pca.fit_transform(df)
 if subplot == 132:
X_reduced_rbf = X_reduced
plt.subplot(subplot)
 plt.title(title, fontsize=14)
 plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=colorlist, cmap=plt.cm.hot)
 plt.xlabel("$z_1$", fontsize=18)
 if subplot == 131:
plt.ylabel("$z_2$", fontsize=18, rotation=0)
 plt.grid(True )<normalization>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
data.head(10 )
lle = LocallyLinearEmbedding(n_components=2, n_neighbors=10, random_state=42)
 lle_X = lle.fit_transform(df )<choose_model_class>
categorical_features = ['Sex', 'SmokingStatus']
features_nn = []
for col in categorical_features:
for mod in data[col].unique() :
features_nn.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data.head(10 )
def model_lstm(input_shape):
inp = Input(shape=(input_shape[1], input_shape[2],))
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(inp)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(input_shape[1] )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[matthews_correlation])
 return model<split>
data['Percent_n'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 data['Age_n'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['base_FVC_n'] =(data['base_FVC'] - data['base_FVC'].min())/(data['base_FVC'].max() - data['base_FVC'].min())
 data['base_week_n'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 features_nn += ['Age_n', 'Percent_n', 'base_week_n', 'base_FVC_n']
print(features_nn)
 data.head(10 )
splits = list(StratifiedKFold(n_splits=N_SPLITS,random_state=42 ).split(X, y))
preds_val = []
y_val = []
for idx,(train_idx, val_idx)in enumerate(splits):
K.clear_session()
print("Beginning fold {}".format(idx+1))
train_X, train_y, val_X, val_y = X[train_idx], y[train_idx], X[val_idx], y[val_idx]
model = model_lstm(train_X.shape)
 ckpt = ModelCheckpoint('weights_{}.h5'.format(idx), save_best_only=True, save_weights_only=True, verbose=1, monitor='val_matthews_correlation', mode='max')
 model.fit(train_X, train_y, batch_size=128, epochs=50, validation_data=[val_X, val_y], callbacks=[ckpt])
 model.load_weights('weights_{}.h5'.format(idx))
preds_val.append(model.predict(val_X, batch_size=512))
y_val.append(val_y)
 preds_val = np.concatenate(preds_val)[...,0]
y_val = np.concatenate(y_val)
 preds_val.shape, y_val.shape<compute_train_metric>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
train.shape, test.shape, sub.shape
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = K.eval(matthews_correlation(y_true.astype(np.float64),(y_proba > threshold ).astype(np.float64)))
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'matthews_correlation': best_score}
return search_result<compute_test_metric>
C1, C2 = tf.constant(70, dtype="float32"), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip * sq2)
 return backend.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.5, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return backend.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
best_threshold = threshold_search(y_val, preds_val)['threshold']<load_from_csv>
def make_model() :
inp = L.Input(len(features_nn), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(inp)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(inp, preds, name="NeuralNet")
 model.compile(loss=mloss(0.64),
optimizer=tf.keras.optimizers.Adam(lr=0.1, decay=0.01),
metrics=[score])
 return model
model = make_model()
model.summary()
meta_test = pd.read_csv('.. /input/metadata_test.csv' )<drop_column>
X_train = train[features_nn].values
X_test = sub[features_nn].values
y_train = train['FVC'].values
oof_train = np.zeros(( X_train.shape[0], 3))
y_preds = np.zeros(( X_test.shape[0], 3))
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<define_variables>
BATCH_SIZE = 128
EPOCHS = 804
NFOLD = 5
kf = KFold(n_splits=NFOLD )
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )<concatenate>
%%time
for fold_id,(tr_idx, va_idx)in enumerate(kf.split(X_train)) :
print(f"FOLD {fold_id+1}")
 model = make_model()
model.fit(X_train[tr_idx], y_train[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(X_train[va_idx], y_train[va_idx]), verbose=0)
 print("train", model.evaluate(X_train[tr_idx], y_train[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", model.evaluate(X_train[va_idx], y_train[va_idx], verbose=0, batch_size=BATCH_SIZE))
oof_train[va_idx] = model.predict(X_train[va_idx], batch_size=BATCH_SIZE, verbose=0)
 y_preds += model.predict(X_test, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape<load_from_csv>
sigma_opt = mean_absolute_error(y_train, oof_train[:, 1])
 sigma_unc = oof_train[:, 2] - oof_train[:, 0]
sigma_mean = np.mean(sigma_unc)
 print(sigma_opt, sigma_mean )
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
print(np.mean(y_train / oof_train[:, 1]))
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict(X_test_input, batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3)
<data_type_conversions>
sub['FVC1'] = y_preds[:, 1]
sub['Confidence1'] = y_preds[:, 2] - y_preds[:, 0]
sub.head(10 )
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
subm = sub[['Patient_Week', 'FVC', 'Confidence', 'FVC1', 'Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
subm.head(10 )
import pandas as pd
import pyarrow.parquet as pq
import os
import numpy as np
from keras.layers import *
from keras.models import *
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras import optimizers
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from keras.callbacks import *
from keras import activations
from keras import regularizers
from keras import initializers
from keras import constraints
from sklearn.preprocessing import MinMaxScaler
from numba import jit
from math import log, floor
from sklearn.neighbors import KDTree
from scipy.signal import periodogram, welch
from keras.engine import Layer
from keras.engine import InputSpec
from keras.objectives import categorical_crossentropy
from keras.objectives import sparse_categorical_crossentropy 
<define_variables>
org_test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(org_test)) :
subm.loc[subm['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'FVC'] = org_test.FVC[i]
subm.loc[subm['Patient_Week']==org_test.Patient[i]+'_'+str(org_test.Weeks[i]), 'Confidence'] = 70
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False)
 reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
N_SPLITS = 5
sample_size = 800000<compute_test_metric>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df = df1[['Patient_Week']].copy()
df['FVC'] = 0.2*df1['FVC'] + 0.8*df2['FVC']
df['Confidence'] = 0.0*df1['Confidence'] + 1.0*df2['Confidence']
df.head()
<choose_model_class><EOS> 
df.to_csv('submission.csv', index=False )