Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<feature_engineering>
tqdm.pandas(tqdm_notebook)
 warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 IS_LOCAL = False
if(IS_LOCAL):
PATH=".. /input/santander-value-prediction-challenge/"
else:
PATH=".. /input/"
print(os.listdir(PATH))
def expandFVC_Percent(DS):
DS['FVC_Percent'] = DS['FVC'] / DS['Percent']
return DS<merge>
train = pd.read_csv(PATH+"train.csv")
 test = pd.read_csv(PATH+"test.csv" )
def expandFE(data_set):
data_set['MinWeeks'] = data_set.groupby('Patient')['Weeks'].transform('min')
 sub_tr = data_set.loc[data_set['Weeks']==data_set['MinWeeks']]
sub_tr['MinFVC'] = sub_tr['FVC']
sub_tr = sub_tr[['Patient', 'MinFVC']]
merged = data_set.merge(sub_tr, on='Patient', how='outer')
 data_set=merged
data_set['MinFVC'] =(data_set['MinFVC'] - data_set['MinFVC'].min())/(data_set['MinFVC'].max() -data_set['MinFVC'].min())
 print(data_set.sample(5))
return data_set<concatenate>
NLAGS = 29
train = expandFE(train)
<define_variables>
all_cols = [f for f in train.columns if f not in ["ID", "target"]]
y = np.log1p(train["target"] ).values
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1',
'15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9',
'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b',
'62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992',
'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd',
'703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2',
'190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98']
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=8):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return math.ceil(len(self.keys)/ float(self.batch_size))
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
self.train_data[k].sort()
mx_len = len(self.train_data[k])
 i = np.random.choice(self.train_data[k][mx_len//3:mx_len//3*2], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x, a, tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
def _get_leak(df, cols, lag=0):

 series_str = df[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 series_shifted_str = df[cols].shift(lag+2, axis=1)[cols[lag+2:]].apply(lambda x: "_".join(x.round(2 ).astype(str)) , axis=1)
 target_rows = series_shifted_str.progress_apply(lambda x: np.where(x == series_str)[0])
 target_vals = target_rows.apply(lambda x: df.loc[x[0], cols[lag]] if len(x)==1 else 0)
 return target_vals
def get_all_leak(df, cols=None, nlags=15):

 df = df.copy()
for i in range(nlags):
print("Processing lag {}".format(i))
df["leaked_target_"+str(i)] = _get_leak(df, cols, i)
 return df
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(5,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 return model<load_pretrained>
test["target"] = train["target"].mean()
efn_path = ".. /input/trainefn/efn.ckpt"
model.load_weights(efn_path )<compute_test_metric>
all_df = pd.concat([train[["ID", "target"] + cols], test[["ID", "target"]+ cols]] ).reset_index(drop=True)
 all_df.head()
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric)
 subs = []
q = -1<load_from_csv>
all_df = get_all_leak(all_df, cols=cols, nlags=NLAGS )
q = -1
with open('.. /input/trainefn/q.txt', 'r')as f:
q=float(f.readlines() [0].strip())
 print(q )<load_from_csv>
leaky_cols = ["leaked_target_"+str(i)for i in range(NLAGS)]
train = train.join(all_df.set_index("ID")[leaky_cols], on="ID", how="left")
 test = test.join(all_df.set_index("ID")[leaky_cols], on="ID", how="left")
 train[["target"]+leaky_cols].head(10 )
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}<save_to_csv>
train["nz_mean"] = train[all_cols].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1)
 test["nz_mean"] = test[all_cols].apply(lambda x: np.expm1(np.log1p(x[x!=0] ).mean()), axis=1 )
N = len(subs)
 print(N)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N)
 sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False)
 img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
print(img_sub.sample(5))<load_from_csv>
train["compiled_leak"] = 0
test["compiled_leak"] = 0
for i in range(NLAGS):
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "leaked_target_"+str(i)]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "leaked_target_"+str(i)]
print("Leak values found in train and test ", sum(train["compiled_leak"] > 0), sum(test["compiled_leak"] > 0))
print("% of correct leaks values in train ", sum(train["compiled_leak"] == train["target"])/sum(train["compiled_leak"] > 0))
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub])
 print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
train.loc[train["compiled_leak"] == 0, "compiled_leak"] = train.loc[train["compiled_leak"] == 0, "nz_mean"]
test.loc[test["compiled_leak"] == 0, "compiled_leak"] = test.loc[test["compiled_leak"] == 0, "nz_mean"]
np.sqrt(mean_squared_error(y, np.log1p(train["compiled_leak"] ).fillna(14.49)))
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min')
 base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True)
 data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
 data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 data['FVC_Percent'] = data['FVC'] / data['Percent']
FE += ['age','percent','week','BASE']
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
print(FE )<train_on_grid>
sub1 = test[["ID"]]
sub1["target"] = test["compiled_leak"]
y = tr['FVC'].values
z = tr[FE].values
print(z.shape)
 ze = sub[FE].values
print(ze.shape)
 nh = z.shape[1]
print(nh)
 pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
net = make_model(nh)
 print(net.summary())
 print(net.count_params())
 NFOLD = 5
kf = KFold(n_splits=NFOLD )<train_model>
NUMBER_KFOLDS = 5
NFOLDS = 5
MAX_ROUNDS = 3000
EARLY_STOP = 100
VERBOSE_EVAL = 200
cnt = 0
EPOCHS = 800
y = y.astype(np.float32)
 for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD<create_dataframe>
train = pd.read_csv(PATH+"train.csv")
 test = pd.read_csv(PATH+"test.csv")
 all_cols = [c for c in train.columns if c not in ['ID', 'target']]
leak_col = []
for c in all_cols:
leak1 = np.sum(( train[c]==train['target'] ).astype(int))
leak2 = np.sum(((( train[c] - train['target'])/ train['target'])< 0.05 ).astype(int))
if leak1 > 30 and leak2 > 3500:
leak_col.append(c )
df_issa2 = df.copy()<save_to_csv>
col = list(leak_col)
 train_lk = train[col + ['ID', 'target']]
test_lk = test[col + ['ID']]
df_issa1.to_csv('df_issa1.csv', index=False)
 df_issa2.to_csv('df_issa2.csv', index=False )<prepare_output>
for df in [train_lk, test_lk]:
df["nz_mean"] = df[col].apply(lambda x: x[x!=0].mean() , axis=1)
 df["nz_max"] = df[col].apply(lambda x: x[x!=0].max() , axis=1)
 df["nz_min"] = df[col].apply(lambda x: x[x!=0].min() , axis=1)
 df["ez"] = df[col].apply(lambda x: len(x[x==0]), axis=1)
 df["mean"] = df[col].apply(lambda x: x.mean() , axis=1)
 df["max"] = df[col].apply(lambda x: x.max() , axis=1)
 df["min"] = df[col].apply(lambda x: x.min() , axis=1)
 df["kurtosis"] = df[col].apply(lambda x: x.kurtosis() , axis=1)
 col += ['nz_mean', 'nz_max', 'nz_min', 'ez', 'mean', 'max', 'min', 'kurtosis']
x = 0.8
df = df_issa1.copy()
df['FVC'] = x*df_issa1['FVC'] +(1-x)*df_issa2['FVC']
df['Confidence'] = x*df_issa1['Confidence'] +(1-x)*df_issa2['Confidence']
df.head()<save_to_csv>
for i in range(2, 100):
train_lk['index'+str(i)] =(( train_lk.index + 2)% i == 0 ).astype(int)
 test_lk['index'+str(i)] =(( test_lk.index + 2)% i == 0 ).astype(int)
 col.append('index'+str(i))
df.to_csv('submission.csv', index=False )<load_from_csv>
test_lk = pd.merge(test_lk, sub1, how='left', on='ID', )
%matplotlib inline
pd.set_option('display.precision', 5)
 training = pd.read_csv(".. /input/train.csv", index_col=0)
 test = pd.read_csv(".. /input/test.csv", index_col=0)
 print(training.shape)
 print(test.shape)
 X = training.iloc[:,:-1]
y = training.TARGET<import_modules>
train_lk = train_lk.replace(0, np.nan)
 test_lk = test_lk.replace(0, np.nan)
 train_lk = pd.concat(( train_lk, test_lk), axis=0, ignore_index=True )
import xgboost as xgb
from sklearn.cross_validation import train_test_split<train_model>
test_lk['target'] = 0.0
folds = NFOLDS
for fold in range(folds):
x1, x2, y1, y2 = model_selection.train_test_split(train_lk[col],
np.log1p(train_lk.target.values),
test_size=0.20,
random_state=fold)
 params = {'learning_rate': 0.02,
'max_depth': 7,
'boosting': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'is_training_metric': True,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'seed':fold}
model = lgb.train(params,
lgb.Dataset(x1, label=y1),
MAX_ROUNDS,
lgb.Dataset(x2, label=y2),
verbose_eval=VERBOSE_EVAL,
early_stopping_rounds=EARLY_STOP)
 test_lk['target'] += np.expm1(model.predict(test_lk[col],
num_iteration=model.best_iteration))
test_lk['target'] /= folds
sub1 = test_lk[['ID', 'target']]
<predict_on_test><EOS> 
test_lk[['ID', 'target']].to_csv('submission.csv', index=False )