Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<merge>
SQRT2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
build_model, get_cosine_annealing_lr_callback, get_fold_indices, DataGenerator, make_lungmask)
 Laplace_metric, Laplace_log_likelihood, experimental_loss_function)
 WANDB = False
SUBMIT = True
TRAIN_ON_BACKWARD_WEEKS = False
PSEUDO_TEST_PATIENTS = 0
df_train = pd.merge(df_train, df_train_old, on = 'card_id')
 df_test = pd.merge(df_test,df_test_old, on = 'card_id')
<drop_column>
if SUBMIT:
PSEUDO_TEST_PATIENTS = 0
WANDB = False
del df_train_old, df_test_old<drop_column>
if WANDB:
user_secrets = UserSecretsClient()
wandb_key = user_secrets.get_secret("wandb_key")
 assert wandb_key, "Please create a key.txt or Kaggle Secret with your W&B API key"
!pip install wandb -qqq --upgrade
!wandb login $wandb_key
target = df_train['target']
del df_train['target'] 
<set_options>
PREDICT_SLOPE = False
USE_IMAGES = False
APPLY_LUNGMASK = False
DIM = 224
IMG_FEATURES = 22
EFFNET = 0
USE_THREE_LAYERS = True
OPTIMAL_SIGMA_LOSS = False
COSINE_CYCLES = 5
DROP_OUT_RATE = 0
DROP_OUT_LAYERS = []
L2_REGULARIZATION = False
REGULARIZATION_CONSTANT = 0.0001
gc.collect()<create_dataframe>
FOLDS = 5
BATCH_SIZE = 128
NUMBER_FEATURES = 10
df_oof = pd.DataFrame()
df_oof['card_id']= df_train['card_id']
df_oof['target'] = target<init_hyperparams>
TTA_STEPS = 1
TTA_MULTIPLIER = 0
HIDDEN_LAYERS = [32,32]
NOISE_SDS = [10,10,1] + [500, 0, 10] + [0.25] + 3*[0.25]
GAUSSIAN_NOISE_FVC_CORRELATED = True
ADD_NOISE_FVC_TO_PERCENT = True
ACTIVATION_FUNCTION = 'swish'
LOSS_MODIFICATION = 1
BATCH_NORMALIZATION = False
PRE_BATCH_NORMALIZATION = False
BATCH_RENORMALIZATION = False
EPOCHS = 500
INPUT_NORMALIZATION = True
OUTPUT_NORMALIZATION = True
NEGATIVE_NORMALIZATION = False
LEARNING_RATE_SCHEDULER = 'exp'
MAX_LEARNING_RATE = 0.001
EPOCHS_PER_OOM_DECAY = 300
MODEL_NAME = "TestNoPercentNoOverfit"
config = dict(NUMBER_FEATURES = NUMBER_FEATURES, L2_REGULARIZATION = L2_REGULARIZATION, INPUT_NORMALIZATION = INPUT_NORMALIZATION, BATCH_RENORMALIZATION = BATCH_RENORMALIZATION,
ACTIVATION_FUNCTION = ACTIVATION_FUNCTION, DROP_OUT_RATE = DROP_OUT_RATE, OUTPUT_NORMALIZATION = OUTPUT_NORMALIZATION, PRE_BATCH_NORMALIZATION = PRE_BATCH_NORMALIZATION,
EPOCHS = EPOCHS, MAX_LEARNING_RATE = MAX_LEARNING_RATE, LOSS_MODIFICATION = LOSS_MODIFICATION, NOISE_SDS = NOISE_SDS, OPTIMAL_SIGMA_LOSS = OPTIMAL_SIGMA_LOSS,
COSINE_CYCLES = COSINE_CYCLES, MODEL_NAME=MODEL_NAME, LEARNING_RATE_SCHEDULER = LEARNING_RATE_SCHEDULER, PREDICT_SLOPE = PREDICT_SLOPE,
HIDDEN_LAYERS = HIDDEN_LAYERS, REGULARIZATION_CONSTANT = REGULARIZATION_CONSTANT, EPOCHS_PER_OOM_DECAY = EPOCHS_PER_OOM_DECAY,
DROP_OUT_LAYERS = DROP_OUT_LAYERS, BATCH_SIZE = BATCH_SIZE, GAUSSIAN_NOISE_FVC_CORRELATED = GAUSSIAN_NOISE_FVC_CORRELATED, TTA_STEPS = TTA_STEPS,
ADD_NOISE_FVC_TO_PERCENT = ADD_NOISE_FVC_TO_PERCENT, NEGATIVE_NORMALIZATION = NEGATIVE_NORMALIZATION, BATCH_NORMALIZATION = BATCH_NORMALIZATION,
APPLY_LUNGMASK = APPLY_LUNGMASK, USE_IMAGES = USE_IMAGES, DIM = DIM, IMG_FEATURES = IMG_FEATURES, EFFNET = EFFNET, TTA_MULTIPLIER = TTA_MULTIPLIER )
param = {'num_leaves':62,
'min_data_in_leaf': 20,
'objective':'regression',
'max_depth': 8,
'learning_rate': 0.001,
"min_child_samples": 20,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9 ,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": 4,
"random_state": 4590}
for i in range(1):
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=4590)
 oof = np.zeros(len(df_train))
predictions = np.zeros(len(df_test))
feature_importance_df = pd.DataFrame()
list_regressor = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(df_train,df_train['outliers'].values)) :
print("fold {}".format(fold_))
trn_data = lgb.Dataset(df_train.iloc[trn_idx][df_train_columns], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(df_train.iloc[val_idx][df_train_columns], label=target.iloc[val_idx])
 num_round = 50000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=500, early_stopping_rounds = 500)
 oof[val_idx] = clf.predict(df_train.iloc[val_idx][df_train_columns], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = df_train_columns
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(df_test[df_train_columns], num_iteration=clf.best_iteration)/ folds.n_splits
list_regressor.append(clf)
 print(np.sqrt(mean_squared_error(oof, target)))
 df_oof['oof'] =oof
df_oof.to_csv("oof.csv" )<save_to_csv>
if SUBMIT:
test_data, submission = get_test_data(".. /input/osic-pulmonary-fibrosis-progression/test.csv")
 test_data["Percent"] = 0
train_data, train_images, train_labels = get_train_data('.. /input/osic-pulmonary-fibrosis-progression/train.csv', PSEUDO_TEST_PATIENTS, TRAIN_ON_BACKWARD_WEEKS, USE_IMAGES, APPLY_LUNGMASK, DIM)
 train_data["Percent"] = 0
np.save("train_data.npy", train_data.to_numpy())
 np.save("train_images.npy", np.array(train_images))
np.save("train_labels.npy", train_labels.to_numpy())
 if PSEUDO_TEST_PATIENTS > 0:
test_data, test_check = get_pseudo_test_data('.. /input/osic-pulmonary-fibrosis-progression/train.csv', PSEUDO_TEST_PATIENTS, INPUT_NORMALIZATION )
fip = best_features.sort_values(by="importance",ascending=False)
 plt.figure(figsize=(20,40))
imp = fip.groupby('Feature')['importance'].mean().sort_values(ascending=True)
 imp.to_csv("feature_importance.csv")
 imp.plot('barh')
 plt.show()<sort_values>
fold_pos = get_fold_indices(FOLDS, train_data)
 print(fold_pos )
Se_imp = fip.groupby('Feature')['importance'].mean().sort_values(ascending=False)
 print(list(Se_imp.index))<save_to_csv>
predictions = []
for fold in range(FOLDS):
train_ID = list(range(fold_pos[0],fold_pos[fold])) + list(range(fold_pos[fold+1],fold_pos[-1]))
val_ID = list(range(fold_pos[fold], fold_pos[fold+1]))
training_generator = DataGenerator(train_ID, config)
 validation_generator = DataGenerator(val_ID, config, validation = True)
 model = build_model(config)
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 callbacks = [sv]
if LEARNING_RATE_SCHEDULER == 'exp':
callbacks.append(get_exponential_decay_lr_callback(config))
if LEARNING_RATE_SCHEDULER == 'cos':
callbacks.append(get_cosine_annealing_lr_callback(config))
print(fold+1, "of", FOLDS)
 if WANDB:
name = MODEL_NAME + '-F{}'.format(fold+1)
 config.update({'fold': fold+1})
 wandb.init(project="pulfib", name = name, config=config)
 wandb_cb = WandbCallback()
callbacks.append(wandb_cb)
 history = model.fit(training_generator, validation_data = validation_generator, epochs = EPOCHS,
verbose = 0, callbacks = callbacks)
 if SUBMIT or PSEUDO_TEST_PATIENTS > 0:
model.load_weights('fold-%i.h5'%fold)
 TTA_test_data = TTA_on_test(test_data.to_numpy() , config)
 for j in range(TTA_STEPS):
predictions.append(model.predict(TTA_test_data[:,:,j], batch_size = 256))
if WANDB:
wandb.join()
for i in range(1):
sub_df = pd.DataFrame({"card_id":df_test["card_id"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<define_variables>
if SUBMIT:
predictions = np.abs(predictions)
 predictions[:,:,1] = np.power(predictions[:,:,1],2)
 predictions = np.mean(predictions, axis = 0)
 predictions[:,1] = np.power(predictions[:,1],0.5)
 for i in range(1,len(test_data)+1):
submission.loc[i,"FVC"] = predictions[i-1,0]
submission.loc[i, "Confidence"] = predictions[i-1,1]
submission.to_csv("submission.csv", index = False)
<import_modules><EOS> 
if PSEUDO_TEST_PATIENTS > 0:
quadraticmeans = []
for j in range(0,11):
result = []
for i in range(-20,20):
postprocess = np.abs(predictions[j])
 if i == 0:
postprocess[:,:,1] = gmean(postprocess[:,:,1], axis = 0)
 postprocess = np.mean(postprocess, axis = 0)
 else:
postprocess[:,:,1] = np.power(postprocess[:,:,1],i)
 postprocess = np.mean(postprocess, axis = 0)
 postprocess[:,1] = np.power(postprocess[:,1],1/i)
 FVC_true = test_check["TargetFVC"].values
FVC_pred = postprocess[:,0]
sigma = postprocess[:,1]
sigma_clip = np.maximum(np.abs(sigma), 70)
 delta = np.abs(FVC_true - FVC_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 loss =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip * sq2)
 result.append(np.mean(loss))
if i == 2:
quadraticmeans.append(result[-1])
 plt.plot(np.arange(-20,20),result)
 plt.show()
plt.plot(0.1*np.arange(0,11),quadraticmeans)
 plt.show()