Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<compute_test_metric>
pd.set_option('display.max_columns', None)
optimal_values = threshold_search(y_val, preds_val)
 best_threshold = optimal_values['threshold']
best_score = optimal_values['matthews_correlation']<compute_test_metric>
env = 'kaggle'
BATCH_SIZE=128
print("Optimal Threshold : " + str(best_threshold))
print("Best Matthews Correlation : " + str(best_score))<load_from_csv>
if env == 'local':
ROOT = ".. /input"
else:
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
%%time
meta_test = pd.read_csv('.. /input/metadata_test.csv' )<drop_column>
train = pd.read_csv(f"{ROOT}/train.csv")
 train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 test = pd.read_csv(f"{ROOT}/test.csv")
 submission = pd.read_csv(f"{ROOT}/sample_submission.csv" )
meta_test = meta_test.set_index(['signal_id'])
 meta_test.head()<define_variables>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 if env == 'local':
tf.random.set_random_seed(seed)
 else:
tf.random.set_seed(seed)
 seed_everything(42 )
%%time
first_sig = meta_test.index[0]
n_parts = 10
max_line = len(meta_test)
 part_size = int(max_line / n_parts)
 last_part = max_line % n_parts
print(first_sig, n_parts, max_line, part_size, last_part, n_parts * part_size + last_part)
 start_end = [[x, x+part_size] for x in range(first_sig, max_line + first_sig, part_size)]
start_end = start_end[:-1] + [[start_end[-1][0], start_end[-1][0] + last_part]]
print(start_end)
 X_test = []
for start, end in start_end:
subset_test = pq.read_pandas('.. /input/test.parquet', columns=[str(i)for i in range(start, end)] ).to_pandas()
for i in tqdm(subset_test.columns):
id_measurement, phase = meta_test.loc[int(i)]
subset_test_col = subset_test[i]
subset_trans = transform_ts(subset_test_col)
 X_test.append([i, id_measurement, phase, subset_trans] )<concatenate>
def laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):

 sd_clipped = np.maximum(confidence, 70)
 delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)
 metric = - np.sqrt(2)* delta / sd_clipped - np.log(np.sqrt(2)* sd_clipped)
 if return_values:
return metric
else:
return np.mean(metric )
X_test_input = np.asarray([np.concatenate([X_test[i][3],X_test[i+1][3], X_test[i+2][3]], axis=1)for i in range(0,len(X_test), 3)])
 np.save("X_test.npy",X_test_input)
 X_test_input.shape<normalization>
all_patient_ids = train['Patient'].unique()
patient_slopes_df = pd.DataFrame()
patient_slopes_df['Patient'] = all_patient_ids
for i in ['Percent','FVC']:
slopes = []
intercepts = []
for patient_id in all_patient_ids:
patient_df = train[train['Patient']==patient_id]
x = patient_df['Weeks'].to_numpy()
y = patient_df[i].to_numpy()
m, b = np.polyfit(x, y, 1)
 slopes.append(m)
 intercepts.append(b)
 patient_slopes_df['Slope_'+i] = slopes
patient_slopes_df['Intercept_'+i] = intercepts
mean_slope_percent = patient_slopes_df['Slope_Percent'].mean()
mean_slope_fvc = patient_slopes_df['Slope_FVC'].mean()
print('mean_slope_percent:',mean_slope_percent)
 print('mean_slope_fvc:',mean_slope_fvc)
 print('')
 print(patient_slopes_df.shape)
 patient_slopes_df.head(3 )
signals = X_test_input.reshape(( len(X_test_input), X_test_input.shape[1]*X_test_input.shape[2]))
features_test = []
for signal in signals:
features_test.append(entropy_and_fractal_dim(signal))<normalization>
submission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])
 submission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
submission = submission[['Patient','Weeks','Confidence','Patient_Week']]
temp_test = test.copy()
temp_test.drop(columns=['Weeks'],inplace=True)
 submission = pd.merge(submission,temp_test, how='left', on=["Patient"])
 del temp_test
submission.head(3 )
features_test = np.array(features_test ).reshape(( len(features_test), 7))
features_test = scaler.transform(features_test)
 np.save("features_test.npy",features_test )<load_from_csv>
temp_test = test.copy()
temp_test['intercept_percent'] = temp_test['Percent'] -(temp_test['Weeks'] * mean_slope_percent)
 submission = pd.merge(submission,temp_test[['Patient','intercept_percent']], how='left', on=["Patient"])
 del temp_test
submission['Percent_mean_slope'] = submission['intercept_percent'] +(submission['Weeks'] * mean_slope_percent)
 submission['Percent'] = submission['Percent_mean_slope']
submission.drop(columns=['intercept_percent','Percent_mean_slope'],inplace=True )
submission = pd.read_csv('.. /input/sample_submission.csv')
 print(len(submission))
submission.head()<predict_on_test>
train['WHERE'] = 'train'
test['WHERE'] = 'test'
submission['WHERE'] = 'submission'
data = train.append([test, submission] )
preds_test = []
for i in range(N_SPLITS):
model.load_weights('weights_{}.h5'.format(i))
pred = model.predict([X_test_input, features_test], batch_size=300, verbose=1)
 pred_3 = []
for pred_scalar in pred:
for i in range(3):
pred_3.append(pred_scalar)
 preds_test.append(pred_3 )<data_type_conversions>
print(train.shape, test.shape, submission.shape, data.shape)
 print(train.Patient.nunique() , test.Patient.nunique() , submission.Patient.nunique() , data.Patient.nunique())
 data.head(3 )
preds_test =(np.squeeze(np.mean(preds_test, axis=0)) > best_threshold ).astype(np.int)
 preds_test.shape<save_to_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='submission','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
submission['target'] = preds_test
submission.to_csv('submission.csv', index=False)
 submission.head()<import_modules>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
<load_from_csv>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'])
 for df in ['ar','hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_dow'] = data[df]['visit_datetime'].dt.dayofweek
data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 data[df] = data[df][data[df]['reserve_datetime_diff'] > data[df]['visit_dow']]
tmp1 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs1', 'reserve_visitors':'rv1'})
 tmp2 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs2', 'reserve_visitors':'rv2'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'])
 data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['doy'] = data['tra']['visit_date'].dt.dayofyear
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['week'] = data['tra']['visit_date'].dt.week
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['doy'] = data['tes']['visit_date'].dt.dayofyear
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['week'] = data['tes']['visit_date'].dt.week
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
unique_stores = data['tes']['air_store_id'].unique()
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True)
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].count().rename(columns={'visitors':'count_observations'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 stores = pd.merge(stores, data['as'], how='left', on=['air_store_id'])
 stores['air_genre_name'] = stores['air_genre_name'].map(lambda x: str(str(x ).replace('/',' ')))
 stores['air_area_name'] = stores['air_area_name'].map(lambda x: str(str(x ).replace('-',' ')))
 lbl = preprocessing.LabelEncoder()
for i in range(10):
stores['air_genre_name'+str(i)] = lbl.fit_transform(stores['air_genre_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_area_name'+str(i)] = lbl.fit_transform(stores['air_area_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_genre_name'] = lbl.fit_transform(stores['air_genre_name'])
 stores['air_area_name'] = lbl.fit_transform(stores['air_area_name'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['day_of_week'] = lbl.fit_transform(data['hol']['day_of_week'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(train, stores, how='inner', on=['air_store_id','dow'])
 test = pd.merge(test, stores, how='left', on=['air_store_id','dow'])
 for df in ['ar','hr']:
train = pd.merge(train, data[df], how='left', on=['air_store_id','visit_date'])
 test = pd.merge(test, data[df], how='left', on=['air_store_id','visit_date'])
 train['id'] = train.apply(lambda r: '_'.join([str(r['air_store_id']), str(r['visit_date'])]), axis=1)
 train['total_reserv_sum'] = train['rv1_x'] + train['rv1_y']
train['total_reserv_mean'] =(train['rv2_x'] + train['rv2_y'])/ 2
train['total_reserv_dt_diff_mean'] =(train['rs2_x'] + train['rs2_y'])/ 2
test['total_reserv_sum'] = test['rv1_x'] + test['rv1_y']
test['total_reserv_mean'] =(test['rv2_x'] + test['rv2_y'])/ 2
test['total_reserv_dt_diff_mean'] =(test['rs2_x'] + test['rs2_y'])/ 2
train['date_int'] = train['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['date_int'] = test['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['var_max_lat'] = train['latitude'].max() - train['latitude']
train['var_max_long'] = train['longitude'].max() - train['longitude']
test['var_max_lat'] = test['latitude'].max() - test['latitude']
test['var_max_long'] = test['longitude'].max() - test['longitude']
train['lon_plus_lat'] = train['longitude'] + train['latitude']
test['lon_plus_lat'] = test['longitude'] + test['latitude']
lbl = preprocessing.LabelEncoder()
train['air_store_id2'] = lbl.fit_transform(train['air_store_id'])
 test['air_store_id2'] = lbl.transform(test['air_store_id'])
 col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date','visitors']]
train = train.fillna(-1)
 test = test.fillna(-1 )<concatenate>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int)
traintest = pd.concat([train, test])
 value_col = ['holiday_flg','min_visitors','mean_visitors','median_visitors',
'count_observations',
'rs1_x','rv1_x','rs2_x','rv2_x','rs1_y','rv1_y','rs2_y','rv2_y','total_reserv_sum','total_reserv_mean',
'total_reserv_dt_diff_mean','date_int','var_max_lat','var_max_long','lon_plus_lat']
cat_col = ['dow', 'year', 'month', 'air_store_id2', 'air_area_name', 'air_genre_name',
'air_area_name0', 'air_area_name1', 'air_area_name2', 'air_area_name3', 'air_area_name4',
'air_area_name5', 'air_area_name6', 'air_genre_name0', 'air_genre_name1',
'air_genre_name2', 'air_genre_name3', 'air_genre_name4']
nn_col = value_col + cat_col
dummys = []
for col in cat_col:
dummy = pd.get_dummies(traintest[col], drop_first=False)
 dummys.append(dummy.as_matrix())
 dummys = np.concatenate(dummys, axis=1)
 value_scaler = preprocessing.MinMaxScaler()
for vcol in value_col:
traintest[vcol] = value_scaler.fit_transform(traintest[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_value = traintest[value_col].as_matrix()
X = np.concatenate([dummys, X_value], axis=1 )<compute_test_metric>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
data_percent_min = data['Percent'].min()
data_percent_max = data['Percent'].max()
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5<choose_model_class>
train = data.loc[data.WHERE=='train']
test = data.loc[data.WHERE=='test']
submission = data.loc[data.WHERE=='submission']
del data
def get_DAE() :
inputs = Input(( X.shape[1],))
x = Dense(800, activation='relu',bias=False )(inputs)
 x = BatchNormalization()(x)
 x = Dense(400, activation='relu', name="feature",bias=False )(x)
 x = BatchNormalization()(x)
 x = Dense(800, activation='relu',bias=False )(x)
 x = BatchNormalization()(x)
 outputs = Dense(X.shape[1], activation='relu' )(x)
 model = Model(inputs=inputs, outputs=outputs)
 model.compile(optimizer='adam', loss='mse')
 return model
def x_generator(x, batch_size, shuffle=True):
batch_index = 0
n = x.shape[0]
while True:
if batch_index == 0:
index_array = np.arange(n)
 if shuffle:
index_array = np.random.permutation(n)
 current_index =(batch_index * batch_size)% n
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_x = x[index_array[current_index: current_index + current_batch_size]]
yield batch_x
def mix_generator(x, batch_size, swaprate=0.15, shuffle=True):
num_value = X.shape[1]
num_swap = int(num_value * swaprate)
 gen1 = x_generator(x, batch_size, shuffle)
 gen2 = x_generator(x, batch_size, shuffle)
 while True:
batch1 = next(gen1)
 batch2 = next(gen2)
 new_batch = batch1.copy()
for i in range(batch1.shape[0]):
swap_idx = np.random.choice(num_value, num_swap, replace=False)
 new_batch[i, swap_idx] = batch2[i, swap_idx]
yield(new_batch, batch1 )<train_model>
percent_model = "avg_slope"
batch_size = 128
num_epoch = 50
gen = mix_generator(X, batch_size)
 dae = get_DAE()
dae.fit_generator(generator=gen,
steps_per_epoch=np.ceil(X.shape[0] / batch_size),
epochs=num_epoch,
verbose=3, )<choose_model_class>
tr_adj = train[train['Weeks']>=0].groupby(['Patient'],as_index=False)['Weeks'].min()
tr_adj = pd.merge(tr_adj,train,how='left',on=['Patient','Weeks'])
 tr_adj = tr_adj.groupby(['Patient','Weeks'],as_index=False)[['Percent','FVC']].mean()
tr_adj['intercept_percent'] = tr_adj['Percent'] -(tr_adj['Weeks'] * mean_slope_percent)
 tr_adj.rename(columns={'Weeks': 'Weeks_0', 'Percent': 'Percent_0','FVC':'FVC_0'}, inplace=True)
 tr_adj.head()
def get_NN(DAE):
l2_loss = l2(0.01)
 DAE.trainable = False
x = dae.get_layer("feature" ).output
x = Dropout(0.1 )(x)
 x = Dense(1500, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.4 )(x)
 x = Dense(500, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.4 )(x)
 x = Dense(500, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.4 )(x)
 predictions = Dense(1, activation='relu', kernel_regularizer=l2_loss,bias=False )(x)
 model = Model(inputs=dae.input, outputs=predictions)
 model.compile(loss='mse',optimizer='adam')
 return model
def train_generator(x, y, batch_size, shuffle=True):
batch_index = 0
n = x.shape[0]
while True:
if batch_index == 0:
index_array = np.arange(n)
 if shuffle:
index_array = np.random.permutation(n)
 current_index =(batch_index * batch_size)% n
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_x = x[index_array[current_index: current_index + current_batch_size]]
batch_y = y[index_array[current_index: current_index + current_batch_size]]
yield batch_x, batch_y
def test_generator(x, batch_size, shuffle=False):
batch_index = 0
n = x.shape[0]
while True:
if batch_index == 0:
index_array = np.arange(n)
 if shuffle:
index_array = np.random.permutation(n)
 current_index =(batch_index * batch_size)% n
if n >= current_index + batch_size:
current_batch_size = batch_size
batch_index += 1
else:
current_batch_size = n - current_index
batch_index = 0
batch_x = x[index_array[current_index: current_index + current_batch_size]]
yield batch_x
def get_callbacks(save_path):
save_checkpoint = ModelCheckpoint(filepath=save_path, monitor='val_loss', save_best_only=True)
 early_stopping = EarlyStopping(monitor='val_loss',
patience=4,
verbose=2,
min_delta=1e-4,
mode='min')
 Callbacks = [ save_checkpoint, early_stopping]
return Callbacks<split>
train_variant = train.copy()
train_variant = pd.merge(train_variant,tr_adj,how='left',on=['Patient'])
 train_variant['Percent_mean_slope'] = train_variant['intercept_percent'] +(train_variant['Weeks'] * mean_slope_percent)
 if percent_model == "weeks_0":
train_variant['percent'] =(train_variant['Percent_0'] - data_percent_min)/(data_percent_max - data_percent_min)
 if percent_model == "avg_slope":
train_variant['percent'] =(train_variant['Percent_mean_slope'] - data_percent_min)/(data_percent_max - data_percent_min)
 train_variant.drop(columns=['Percent_mean_slope','intercept_percent'],inplace=True)
 print(train_variant.shape)
 train_variant.head()
batch_size = 128
num_epoch = 40
num_fold = 3
Y_train = np.log1p(train['visitors'] ).values
X_train = X[:Y_train.shape[0]]
X_test = X[Y_train.shape[0]:]
y_test = np.zeros([num_fold, X_test.shape[0]])
 y_valid = np.zeros([X_train.shape[0]])
 folds = list(KFold(n_splits=num_fold, shuffle=True, random_state=42 ).split(X_train))
for j,(ids_train_split, ids_valid_split)in enumerate(folds):
print("fold", j+1, "==================")
 model = get_NN(dae)
 gen_train = train_generator(X_train[ids_train_split], Y_train[ids_train_split], batch_size)
 gen_val = train_generator(X_train[ids_valid_split], Y_train[ids_valid_split], batch_size, shuffle=False)
 gen_val_pred = test_generator(X_train[ids_valid_split], batch_size, shuffle=False)
 gen_test_pred = test_generator(X_test, batch_size, shuffle=False)
 callbacks = get_callbacks("weight" + str(j)+ ".hdf5")
 model.fit_generator(generator=gen_train,
steps_per_epoch=np.ceil(ids_train_split.shape[0] / batch_size),
epochs=num_epoch,
verbose=3,
validation_data=gen_val,
validation_steps=np.ceil(ids_valid_split.shape[0] / batch_size), 
)
 y_valid[ids_valid_split] = model.predict_generator(generator=gen_val_pred,
steps=np.ceil(ids_valid_split.shape[0] / batch_size)) [:,0]
y_test[j] = model.predict_generator(generator=gen_test_pred,
steps=np.ceil(X_test.shape[0] / batch_size)) [:,0]
score = RMSLE(y_valid, Y_train)
 print("valid score", score)
 y_test_mean = np.mean(y_test, axis=0)
<save_to_csv>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
id = pd.read_csv(".. /input/sample_submission.csv")['id']
submission = pd.DataFrame({'id': id, 'visitors': np.expm1(y_test_mean)})
 submission.to_csv('submission_dae.csv', index=False )<define_variables>
y_train = train['FVC'].values.astype(np.float32)
 x_train = train[FE].values.astype(np.float32)
 x_train_variant = train_variant[FE].values.astype(np.float32)
 x_test = test[FE].values.astype(np.float32)
 x_submission = submission[FE].values.astype(np.float32)
 pred_test = np.zeros(( x_test.shape[0], 3))
pred_train = np.zeros(( x_train.shape[0], 3))
pred_submission = np.zeros(( x_submission.shape[0], 3))
NWEEKS_TO_EXCLUDE = 2<import_modules>
num_inputs = x_train.shape[1]
net = make_model(num_inputs)
 print(net.summary())
 print(net.count_params() )
<load_from_csv>
NFOLD = 5
repeats = 10
data = {
'tra': pd.read_csv('.. /input/air_visit_data.csv'),
'as': pd.read_csv('.. /input/air_store_info.csv'),
'hs': pd.read_csv('.. /input/hpg_store_info.csv'),
'ar': pd.read_csv('.. /input/air_reserve.csv'),
'hr': pd.read_csv('.. /input/hpg_reserve.csv'),
'id': pd.read_csv('.. /input/store_id_relation.csv'),
'tes': pd.read_csv('.. /input/sample_submission.csv'),
'hol': pd.read_csv('.. /input/date_info.csv' ).rename(columns={'calendar_date':'visit_date'})
 }
data['hr'] = pd.merge(data['hr'], data['id'], how='inner', on=['hpg_store_id'])
 for df in ['ar','hr']:
data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
 data[df]['visit_dow'] = data[df]['visit_datetime'].dt.dayofweek
data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
 data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
data[df]['reserve_datetime_diff'] = data[df].apply(lambda r:(r['visit_datetime'] - r['reserve_datetime'] ).days, axis=1)
 ne = 7*(NWEEKS_TO_EXCLUDE - 1)
 data[df] = data[df][data[df]['reserve_datetime_diff'] > data[df]['visit_dow']+ne]
tmp1 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs1', 'reserve_visitors':'rv1'})
 tmp2 = data[df].groupby(['air_store_id','visit_datetime'], as_index=False)[['reserve_datetime_diff', 'reserve_visitors']].mean().rename(columns={'visit_datetime':'visit_date', 'reserve_datetime_diff': 'rs2', 'reserve_visitors':'rv2'})
 data[df] = pd.merge(tmp1, tmp2, how='inner', on=['air_store_id','visit_date'])
 data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
 data['tra']['dow'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x ).split('_')[2])
 data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
 data['tes']['dow'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
unique_stores = data['tes']['air_store_id'].unique()
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores, 'dow': [i]*len(unique_stores)})for i in range(7)], axis=0, ignore_index=True ).reset_index(drop=True)
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].max().rename(columns={'visitors':'max_visitors'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 tmp = data['tra'].groupby(['air_store_id','dow'], as_index=False)['visitors'].count().rename(columns={'visitors':'count_observations'})
 stores = pd.merge(stores, tmp, how='left', on=['air_store_id','dow'])
 stores = pd.merge(stores, data['as'], how='left', on=['air_store_id'])
 stores['air_genre_name'] = stores['air_genre_name'].map(lambda x: str(str(x ).replace('/',' ')))
 stores['air_area_name'] = stores['air_area_name'].map(lambda x: str(str(x ).replace('-',' ')))
 lbl = preprocessing.LabelEncoder()
for i in range(10):
stores['air_genre_name'+str(i)] = lbl.fit_transform(stores['air_genre_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_area_name'+str(i)] = lbl.fit_transform(stores['air_area_name'].map(lambda x: str(str(x ).split(' ')[i])if len(str(x ).split(' ')) >i else ''))
stores['air_genre_name'] = lbl.fit_transform(stores['air_genre_name'])
 stores['air_area_name'] = lbl.fit_transform(stores['air_area_name'])
 data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
 data['hol']['day_of_week'] = lbl.fit_transform(data['hol']['day_of_week'])
 data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
 test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
 train = pd.merge(train, stores, how='inner', on=['air_store_id','dow'])
 test = pd.merge(test, stores, how='left', on=['air_store_id','dow'])
 for df in ['ar','hr']:
train = pd.merge(train, data[df], how='left', on=['air_store_id','visit_date'])
 test = pd.merge(test, data[df], how='left', on=['air_store_id','visit_date'])
 train['id'] = train.apply(lambda r: '_'.join([str(r['air_store_id']), str(r['visit_date'])]), axis=1)
 train['total_reserv_sum'] = train['rv1_x'] + train['rv1_y']
train['total_reserv_mean'] =(train['rv2_x'] + train['rv2_y'])/ 2
train['total_reserv_dt_diff_mean'] =(train['rs2_x'] + train['rs2_y'])/ 2
test['total_reserv_sum'] = test['rv1_x'] + test['rv1_y']
test['total_reserv_mean'] =(test['rv2_x'] + test['rv2_y'])/ 2
test['total_reserv_dt_diff_mean'] =(test['rs2_x'] + test['rs2_y'])/ 2
train['date_int'] = train['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 test['date_int'] = test['visit_date'].apply(lambda x: x.strftime('%Y%m%d')).astype(int)
 train['var_max_lat'] = train['latitude'].max() - train['latitude']
train['var_max_long'] = train['longitude'].max() - train['longitude']
test['var_max_lat'] = test['latitude'].max() - test['latitude']
test['var_max_long'] = test['longitude'].max() - test['longitude']
train['lon_plus_lat'] = train['longitude'] + train['latitude']
test['lon_plus_lat'] = test['longitude'] + test['latitude']
lbl = preprocessing.LabelEncoder()
train['air_store_id2'] = lbl.fit_transform(train['air_store_id'])
 test['air_store_id2'] = lbl.transform(test['air_store_id'])
 col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date','visitors']]
train = train.fillna(-1)
 test = test.fillna(-1 )<compute_test_metric>
%%time
preds_df = pd.DataFrame()
conf_df = pd.DataFrame()
preds_test_df = pd.DataFrame()
conf_test_df = pd.DataFrame()
preds_submission_df = pd.DataFrame()
conf_submission_df = pd.DataFrame()
for random_state in range(repeats):
print('repeat:',random_state)
 pred_test = np.zeros(( x_test.shape[0], 3))
pred_train = np.zeros(( x_train.shape[0], 3))
pred_submission = np.zeros(( x_submission.shape[0], 3))
kf = KFold(n_splits=NFOLD,shuffle=True,random_state=random_state)
 cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(x_train):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(num_inputs)
 net.fit(x_train[tr_idx], y_train[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(x_train[val_idx], y_train[val_idx]), verbose=0)
 print("train", net.evaluate(x_train[tr_idx], y_train[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(x_train[val_idx], y_train[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict train-val...")
 pred_train[val_idx] = net.predict(x_train_variant[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pred_test += net.predict(x_test, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
print("predict submission...")
 pred_submission += net.predict(x_submission, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
sigma_opt = mean_absolute_error(y_train, pred_train[:, 1])
 unc = pred_train[:,2] - pred_train[:, 0]
sigma_mean = np.mean(unc)
 print('----------------------------')
 print(sigma_opt, sigma_mean)
 preds_df[random_state] = pred_train[:, 1]
conf_df[random_state] = unc
preds_test_df[random_state] = pred_test[:, 1]
conf_test_df[random_state] = pred_test[:,2] - pred_test[:, 0]
preds_submission_df[random_state] = pred_submission[:, 1]
conf_submission_df[random_state] = pred_submission[:,2] - pred_submission[:, 0]
def RMSLE(y, pred):
return metrics.mean_squared_error(y, pred)**0.5<define_variables>
preds_df['mean'] = preds_df.iloc[:,0:repeats].mean(axis=1)
 conf_df['mean'] = conf_df.iloc[:,0:repeats].mean(axis=1)
 preds_df['median'] = preds_df.iloc[:,0:repeats].median(axis=1)
 conf_df['median'] = conf_df.iloc[:,0:repeats].median(axis=1)
 preds_test_df['mean'] = preds_test_df.iloc[:,0:repeats].mean(axis=1)
 conf_test_df['mean'] = conf_test_df.iloc[:,0:repeats].mean(axis=1)
 preds_test_df['median'] = preds_test_df.iloc[:,0:repeats].median(axis=1)
 conf_test_df['median'] = conf_test_df.iloc[:,0:repeats].median(axis=1)
 preds_submission_df['mean'] = preds_submission_df.iloc[:,0:repeats].mean(axis=1)
 conf_submission_df['mean'] = conf_submission_df.iloc[:,0:repeats].mean(axis=1)
 preds_submission_df['median'] = preds_submission_df.iloc[:,0:repeats].median(axis=1)
 conf_submission_df['median'] = conf_submission_df.iloc[:,0:repeats].median(axis=1 )
value_col = ['holiday_flg','min_visitors','mean_visitors','median_visitors','max_visitors','count_observations',
'rs1_x','rv1_x','rs2_x','rv2_x','rs1_y','rv1_y','rs2_y','rv2_y','total_reserv_sum','total_reserv_mean',
'total_reserv_dt_diff_mean','date_int','var_max_lat','var_max_long','lon_plus_lat']
nn_col = value_col + ['dow', 'year', 'month', 'air_store_id2', 'air_area_name', 'air_genre_name',
'air_area_name0', 'air_area_name1', 'air_area_name2', 'air_area_name3', 'air_area_name4',
'air_area_name5', 'air_area_name6', 'air_genre_name0', 'air_genre_name1',
'air_genre_name2', 'air_genre_name3', 'air_genre_name4']
X = train.copy()
X_test = test[nn_col].copy()
value_scaler = preprocessing.MinMaxScaler()
for vcol in value_col:
X[vcol] = value_scaler.fit_transform(X[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_test[vcol] = value_scaler.transform(X_test[vcol].values.astype(np.float64 ).reshape(-1, 1))
X_train = list(X[nn_col].T.as_matrix())
 Y_train = np.log1p(X['visitors'] ).values
nn_train = [X_train, Y_train]
nn_test = [list(X_test[nn_col].T.as_matrix())]
print("Train and test data prepared" )<choose_model_class>
scores = []
for i in range(repeats):
score = laplace_log_likelihood(y_train, preds_df[i], conf_df[i])
 print('solution',i,':',score)
 scores.append(score)
 print('----------------')
 print('mean score:',np.mean(scores))
def get_nn_complete_model(train, hidden1_neurons=35, hidden2_neurons=15):

 K.clear_session()
air_store_id = Input(shape=(1,), dtype='int32', name='air_store_id')
 air_store_id_emb = Embedding(len(train['air_store_id2'].unique())+ 1, 15, input_shape=(1,),
name='air_store_id_emb' )(air_store_id)
 air_store_id_emb = keras.layers.Flatten(name='air_store_id_emb_flatten' )(air_store_id_emb)
 dow = Input(shape=(1,), dtype='int32', name='dow')
 dow_emb = Embedding(8, 3, input_shape=(1,), name='dow_emb' )(dow)
 dow_emb = keras.layers.Flatten(name='dow_emb_flatten' )(dow_emb)
 month = Input(shape=(1,), dtype='int32', name='month')
 month_emb = Embedding(13, 3, input_shape=(1,), name='month_emb' )(month)
 month_emb = keras.layers.Flatten(name='month_emb_flatten' )(month_emb)
 air_area_name, air_genre_name = [], []
air_area_name_emb, air_genre_name_emb = [], []
for i in range(7):
area_name_col = 'air_area_name' + str(i)
 air_area_name.append(Input(shape=(1,), dtype='int32', name=area_name_col))
tmp = Embedding(len(train[area_name_col].unique()), 3, input_shape=(1,),
name=area_name_col + '_emb' )(air_area_name[-1])
 tmp = keras.layers.Flatten(name=area_name_col + '_emb_flatten' )(tmp)
 air_area_name_emb.append(tmp)
 if i > 4:
continue
area_genre_col = 'air_genre_name' + str(i)
 air_genre_name.append(Input(shape=(1,), dtype='int32', name=area_genre_col))
tmp = Embedding(len(train[area_genre_col].unique()), 3, input_shape=(1,),
name=area_genre_col + '_emb' )(air_genre_name[-1])
 tmp = keras.layers.Flatten(name=area_genre_col + '_emb_flatten' )(tmp)
 air_genre_name_emb.append(tmp)
 air_genre_name_emb = keras.layers.concatenate(air_genre_name_emb)
 air_genre_name_emb = Dense(4, activation='sigmoid', name='final_air_genre_emb' )(air_genre_name_emb)
 air_area_name_emb = keras.layers.concatenate(air_area_name_emb)
 air_area_name_emb = Dense(4, activation='sigmoid', name='final_air_area_emb' )(air_area_name_emb)
 air_area_code = Input(shape=(1,), dtype='int32', name='air_area_code')
 air_area_code_emb = Embedding(len(train['air_area_name'].unique()), 8, input_shape=(1,), name='air_area_code_emb' )(air_area_code)
 air_area_code_emb = keras.layers.Flatten(name='air_area_code_emb_flatten' )(air_area_code_emb)
 air_genre_code = Input(shape=(1,), dtype='int32', name='air_genre_code')
 air_genre_code_emb = Embedding(len(train['air_genre_name'].unique()), 5, input_shape=(1,),
name='air_genre_code_emb' )(air_genre_code)
 air_genre_code_emb = keras.layers.Flatten(name='air_genre_code_emb_flatten' )(air_genre_code_emb)
 holiday_flg = Input(shape=(1,), dtype='float32', name='holiday_flg')
 year = Input(shape=(1,), dtype='float32', name='year')
 min_visitors = Input(shape=(1,), dtype='float32', name='min_visitors')
 mean_visitors = Input(shape=(1,), dtype='float32', name='mean_visitors')
 median_visitors = Input(shape=(1,), dtype='float32', name='median_visitors')
 max_visitors = Input(shape=(1,), dtype='float32', name='max_visitors')
 count_observations = Input(shape=(1,), dtype='float32', name='count_observations')
 rs1_x = Input(shape=(1,), dtype='float32', name='rs1_x')
 rv1_x = Input(shape=(1,), dtype='float32', name='rv1_x')
 rs2_x = Input(shape=(1,), dtype='float32', name='rs2_x')
 rv2_x = Input(shape=(1,), dtype='float32', name='rv2_x')
 rs1_y = Input(shape=(1,), dtype='float32', name='rs1_y')
 rv1_y = Input(shape=(1,), dtype='float32', name='rv1_y')
 rs2_y = Input(shape=(1,), dtype='float32', name='rs2_y')
 rv2_y = Input(shape=(1,), dtype='float32', name='rv2_y')
 total_reserv_sum = Input(shape=(1,), dtype='float32', name='total_reserv_sum')
 total_reserv_mean = Input(shape=(1,), dtype='float32', name='total_reserv_mean')
 total_reserv_dt_diff_mean = Input(shape=(1,), dtype='float32', name='total_reserv_dt_diff_mean')
 date_int = Input(shape=(1,), dtype='float32', name='date_int')
 var_max_lat = Input(shape=(1,), dtype='float32', name='var_max_lat')
 var_max_long = Input(shape=(1,), dtype='float32', name='var_max_long')
 lon_plus_lat = Input(shape=(1,), dtype='float32', name='lon_plus_lat')
 date_emb = keras.layers.concatenate([dow_emb, month_emb, year, holiday_flg])
 date_emb = Dense(5, activation='sigmoid', name='date_merged_emb' )(date_emb)
 cat_layer = keras.layers.concatenate([holiday_flg, min_visitors, mean_visitors,
median_visitors, max_visitors, count_observations, rs1_x, rv1_x,
rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y,
total_reserv_sum, total_reserv_mean, total_reserv_dt_diff_mean,
date_int, var_max_lat, var_max_long, lon_plus_lat,
date_emb, air_area_name_emb, air_genre_name_emb,
air_area_code_emb, air_genre_code_emb, air_store_id_emb])
 m = Dense(hidden1_neurons, name='hidden1',
kernel_initializer=keras.initializers.RandomNormal(mean=0.0,
stddev=0.05, seed=None))(cat_layer)
 m = keras.layers.PReLU()(m)
 m = keras.layers.BatchNormalization()(m)
 m1 = Dense(hidden2_neurons, name='sub1' )(m)
 m1 = keras.layers.PReLU()(m1)
 m = Dense(1, activation='relu' )(m1)
 inp_ten = [
holiday_flg, min_visitors, mean_visitors, median_visitors, max_visitors, count_observations,
rs1_x, rv1_x, rs2_x, rv2_x, rs1_y, rv1_y, rs2_y, rv2_y, total_reserv_sum, total_reserv_mean,
total_reserv_dt_diff_mean, date_int, var_max_lat, var_max_long, lon_plus_lat,
dow, year, month, air_store_id, air_area_code, air_genre_code
]
inp_ten += air_area_name
inp_ten += air_genre_name
model = keras.Model(inp_ten, m)
 model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])
 return model<train_model>
print('mean OOF lap_log:',laplace_log_likelihood(y_train, preds_df['mean'], conf_df['mean']))
print('median OOF lap_log:',laplace_log_likelihood(y_train, preds_df['median'], conf_df['median']))
model1 = ensemble.GradientBoostingRegressor(learning_rate=0.2, random_state=3,
n_estimators=200, subsample=0.8, max_depth =10)
 model2 = neighbors.KNeighborsRegressor(n_jobs=-1, n_neighbors=4)
 model3 = XGBRegressor(learning_rate=0.02, random_state=3, n_estimators=800, subsample=0.8,
colsample_bytree=0.8, max_depth =10)
 model4 = get_nn_complete_model(train, hidden2_neurons=12)
 model1.fit(train[col], np.log1p(train['visitors'].values))
print("Model1 trained")
 model2.fit(train[col], np.log1p(train['visitors'].values))
print("Model2 trained")
 model3.fit(train[col], np.log1p(train['visitors'].values))
print("Model3 trained")
 for i in range(5):
model4.fit(nn_train[0], nn_train[1], epochs=3, verbose=1,
batch_size=256, shuffle=True, validation_split=0.15)
 model4.fit(nn_train[0], nn_train[1], epochs=8, verbose=0,
batch_size=256, shuffle=True)
 print("Model4 trained")
 preds1 = model1.predict(train[col])
 preds2 = model2.predict(train[col])
 preds3 = model3.predict(train[col])
 preds4 = pd.Series(model4.predict(nn_train[0] ).reshape(-1)).clip(0, 6.8 ).values
print('RMSE GradientBoostingRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds1))
print('RMSE KNeighborsRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds2))
print('RMSE XGBRegressor: ', RMSLE(np.log1p(train['visitors'].values), preds3))
print('RMSE NeuralNetwork: ', RMSLE(np.log1p(train['visitors'].values), preds4))
preds1 = model1.predict(test[col])
 preds2 = model2.predict(test[col])
 preds3 = model3.predict(test[col])
 preds4 = pd.Series(model4.predict(nn_test[0] ).reshape(-1)).clip(0, 6.8 ).values
test['visitors'] = 0.15*preds1+0.15*preds2+0.4*preds3+0.3*preds4
test['visitors'] = np.expm1(test['visitors'] ).clip(lower=0.)
sub1 = test[['id','visitors']].copy()
print("Model predictions done.")
<feature_engineering>
temp = train_variant.copy()
temp['prediction'] = preds_df['mean']
temp['confidence'] = conf_df['mean']
temp['for_eval'] = temp['Weeks'] != temp['Weeks_0']
temp = temp[temp['for_eval']==True]
print('mean OOF lap_log:',laplace_log_likelihood(temp['FVC'], temp['prediction'], temp['confidence']))
dfs = { re.search('/([^/\.]*)\.csv', fn ).group(1):
pd.read_csv(fn)for fn in glob.glob('.. /input/*.csv')}
for k, v in dfs.items() : locals() [k] = v
wkend_holidays = date_info.apply(
(lambda x:(x.day_of_week=='Sunday' or x.day_of_week=='Saturday')and x.holiday_flg==1), axis=1)
 date_info.loc[wkend_holidays, 'holiday_flg'] = 0
date_info['weight'] =(( date_info.index + 1)/ len(date_info)) ** 5
visit_data = air_visit_data.merge(date_info, left_on='visit_date', right_on='calendar_date', how='left')
 visit_data.drop('calendar_date', axis=1, inplace=True)
 visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
 wmean = lambda x:(( x.weight * x.visitors ).sum() / x.weight.sum())
 visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg'] ).apply(wmean ).reset_index()
visitors.rename(columns={0:'visitors'}, inplace=True)
 sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
 sample_submission.drop('visitors', axis=1, inplace=True)
 sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
 sample_submission = sample_submission.merge(visitors, on=[
'air_store_id', 'day_of_week', 'holiday_flg'], how='left')
 sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
 sub2 = sample_submission[['id', 'visitors']].copy()
sub2 = sub2.fillna(-1 )<save_to_csv>
p_oof = train_variant.copy()
p_oof['prediction'] = preds_df['mean']
p_oof['confidence'] = conf_df['mean']
p_oof.head(3 )
def final_visitors(x, alt=False):
visitors_x, visitors_y = x['visitors_x'], x['visitors_y']
if x['visitors_y'] == -1:
return visitors_x
else:
return 0.7*visitors_x + 0.3*visitors_y* 1.1
sub_merge = pd.merge(sub1, sub2, on='id', how='inner')
 sub_merge['visitors'] = sub_merge.apply(lambda x: final_visitors(x), axis=1)
 print("Done")
 sub_merge[['id', 'visitors']].to_csv('submission.csv', index=False )<import_modules>
train_adj = p_oof[p_oof['Weeks']>=0].groupby(['Patient'],as_index=False)['Weeks'].min()
train_adj = pd.merge(train_adj,p_oof,how='left',on=['Patient','Weeks'])
 train_adj = train_adj.groupby(['Patient','Weeks'],as_index=False)[['Percent','FVC']].mean()
train_adj.rename(columns={'Weeks': 'First_week'}, inplace=True)
 train_adj.head(3 )
<load_from_csv>
p_oof = pd.merge(p_oof,train_adj[['Patient','First_week']],how='left',on=['Patient'])
 p_oof_week_0 = p_oof[p_oof['Weeks']==p_oof['First_week']][['Patient','FVC','prediction']].copy()
p_oof_week_0.reset_index(inplace=True, drop=True)
 p_oof_week_0['prediction_w0_delta'] = p_oof_week_0['FVC'] - p_oof_week_0['prediction']
p_oof_week_0.head(3 )
<load_from_csv>
p_oof = pd.merge(p_oof,p_oof_week_0[['Patient','prediction_w0_delta']],how='left',on=['Patient'])
 p_oof['predition_adjusted'] = p_oof['prediction'] + p_oof['prediction_w0_delta']
<train_model>
temp = p_oof.copy()
temp['for_eval'] =(temp['Weeks'] != temp['First_week'])
 temp = temp[temp['for_eval']==True]
print('mean OOF lap_log:',laplace_log_likelihood(temp['FVC'], temp['predition_adjusted'], temp['confidence']))
<set_options>
save_oof = False
if save_oof:
check_me = train.copy()
check_me['prediction'] = preds_df['mean']
check_me['confidence'] = conf_df['mean']
check_me.head(3)
 check_me.to_csv("oof_name_here.csv", index=False )
gc.enable()<load_from_csv>
my_sub = submission.copy()
my_sub.reset_index(inplace=True, drop=True)
 my_sub = my_sub[['Patient','Weeks']]
my_sub['FVC'] = preds_submission_df['mean'].values
my_sub['Confidence'] = conf_submission_df['mean'].values
my_sub.head()
<prepare_x_and_y><EOS> 
make_submission = True
if make_submission:
my_sub['Weeks'] = my_sub['Weeks'].astype(str)
 my_sub['Patient_Week'] = my_sub['Patient'] + '_' + my_sub['Weeks']
my_sub = my_sub[['Patient_Week','FVC','Confidence']]
print(my_sub.shape)
 my_sub.head()
my_sub.to_csv("submission.csv", index=False )