Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<sort_values>
HTML('<iframe width="800" height="400" src="https://www.youtube.com/embed/-UeC4MR3PHM?start=0" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>' )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
py.init_notebook_mode(connected=True)
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()<save_to_csv>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
df.to_csv('submission.csv', index=False )<set_options>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
warnings.filterwarnings("ignore" )<import_modules>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import KFold<load_from_csv>
np.sum(train_df['meter_reading_log1p'].values < 0 )
df = pd.read_csv("/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv")
 test = pd.read_csv("/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv")
 sample = pd.read_csv("/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv" )<count_unique_values>
building_meta_df[building_meta_df.site_id == 0]
print(f"Out of 1549 entried there were only {df['Patient'].nunique() } unique patients" )<count_duplicates>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
print("Since the data has dupilcated values let's recheck by dropping duplicates")
 print()
print("
print()
print(df[['Patient','Sex','SmokingStatus']].drop_duplicates() ['Sex'].value_counts())
 print()
print("
print()
print(df[['Patient','Sex','SmokingStatus']].drop_duplicates() ['SmokingStatus'].value_counts() )<feature_engineering>
debug = False
df['Photo count'] = 0
names = df['Patient'].unique()
for name in names:
file = '/kaggle/input/osic-pulmonary-fibrosis-progression/train/'+name
df.loc[df['Patient']==name,'Photo count'] = len(os.listdir(file))
data = df.groupby(by="Patient")["Photo count"].first().reset_index(drop=False)
 data = data.sort_values(['Photo count'] ).reset_index(drop=True)
 data['Photo count'].describe()<define_variables>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
df["weekend"] = df["dayofweek"] >= 5
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(42 )<remove_duplicates>
preprocess(train_df )
df.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'] )<merge>
sample['Patient'] = sample['Patient_Week'].apply(lambda x:x.split('_')[0])
 sample['Weeks'] = sample['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sample = sample[['Patient','Weeks','Confidence','Patient_Week']]
sample = sample.merge(test.drop('Weeks', axis=1), on="Patient" )<concatenate>
df_group = train_df.groupby(['building_id', 'meter'])['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16 )
df['WHERE'] = 'train'
test['WHERE'] = 'val'
sample['WHERE'] = 'test'
data = df.append([test, sample] )<feature_engineering>
building_stats_df = pd.concat([building_mean, building_median, building_min, building_max, building_std], axis=1,
keys=['building_mean', 'building_median', 'building_min', 'building_max', 'building_std'] ).reset_index()
train_df = pd.merge(train_df, building_stats_df, on=['building_id', 'meter'], how='left', copy=False )
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
weather_train_df.isna().sum()
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
weather_test_df = pd.read_feather(root/'weather_test.feather')
 weather = pd.concat([weather_train_df, weather_test_df],ignore_index=True)
 del weather_test_df
weather_key = ['site_id', 'timestamp']
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
gc.collect()<categorify>
temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df
del weather
del temp_skeleton
gc.collect()
data[['Sex_Male','SmokingStatus_Ex-smoker','SmokingStatus_Never smoked']] = pd.get_dummies(data[['Sex','SmokingStatus']],drop_first=True)
 features = ['Percent','Age','min_FVC','base_week']<normalization>
weather_train_df = timestamp_align(weather_train_df)
 weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
for i in features:
scaler = MinMaxScaler()
data[i] = scaler.fit_transform(data[[i]] )<drop_column>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
df = data.loc[data.WHERE=='train'].drop('WHERE',axis=1)
 test = data.loc[data.WHERE=='val'].drop('WHERE',axis=1)
 sample = data.loc[data.WHERE=='test'].drop('WHERE',axis=1 )<define_variables>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
features += ['Sex_Male','SmokingStatus_Ex-smoker','SmokingStatus_Never smoked']<prepare_x_and_y>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
X = df[features].values
y = df['FVC'].values
test_ = sample[features].values
y = y.astype('float64' )<define_variables>
category_cols = ['building_id', 'site_id', 'primary_use']
weather_cols = [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend', 'dayofweek',
'building_median'] + weather_cols
nh = X.shape[1]
pe = np.zeros(( test_.shape[0], 3))
pred = np.zeros(( X.shape[0], 3))<compute_test_metric>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(tf.dtypes.cast(y_true[:, 0], tf.float32)- tf.dtypes.cast(fvc_pred, tf.float32))
delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
e = tf.dtypes.cast(y_true,tf.float32)- tf.dtypes.cast(y_pred,tf.float32)
 v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model<train_model>
optuna.__version__
kf = KFold(n_splits=5)
 cnt = 0
EPOCHS = 800
for tr_idx, val_idx in kf.split(X):
cnt += 1
print(f"FOLD {cnt}")
 model = make_model(nh)
 model.fit(X[tr_idx], y[tr_idx], batch_size=128, epochs=EPOCHS,
validation_data=(X[val_idx], y[val_idx]), verbose=0)
 print("train", model.evaluate(X[tr_idx], y[tr_idx], verbose=0, batch_size=128))
print("val", model.evaluate(X[val_idx], y[val_idx], verbose=0, batch_size=128))
print("predict val...")
 pred[val_idx] = model.predict(X[val_idx], batch_size=128, verbose=0)
 print("predict test...")
 pe += model.predict(test_, batch_size=128, verbose=0)/ 5<compute_test_metric>
def objective(trial: Trial, fast_check=True, target_meter=0, return_info=False):
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models = []
valid_score = 0
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(trial, train_data, valid_data, cat_features=category_cols,
num_rounds=1000)
 y_valid_pred_total[valid_idx] = y_pred_valid
models.append(model)
 gc.collect()
valid_score += log["valid/l2"]
if fast_check:
break
valid_score /= len(models)
 if return_info:
return valid_score, models, y_pred_valid, y_train
else:
return valid_score
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )<prepare_output>
def fit_lgbm(trial, train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
'objective': 'regression',
'learning_rate': 0.1,
"boosting": "gbdt",
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
"bagging_freq": 5,
"bagging_fraction": trial.suggest_uniform('bagging_fraction', 0.1, 1.0),
"feature_fraction": trial.suggest_uniform('feature_fraction', 0.4, 1.0),
"metric": metric,
"verbosity": -1,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/l2': model.best_score['training']['l2'],
'valid/l2': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
sample['FVC1'] = 0.996*pe[:, 1]
sample['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sample[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()<feature_engineering>
study = optuna.create_study()
study.optimize(objective, n_trials=10 )
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']<load_from_csv>
def fit_lgbm_with_pruning(trial, train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {
'num_leaves': trial.suggest_int('num_leaves', 2, 256),
'objective': 'regression',
'learning_rate': 0.1,
"boosting": "gbdt",
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
"bagging_freq": 5,
"bagging_fraction": trial.suggest_uniform('bagging_fraction', 0.1, 1.0),
"feature_fraction": trial.suggest_uniform('feature_fraction', 0.4, 1.0),
"metric": metric,
"verbosity": -1,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
pruning_callback = optuna.integration.LightGBMPruningCallback(trial, 'l2', valid_name='valid_1')
 print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop,
callbacks=[pruning_callback])
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/l2': model.best_score['training']['l2'],
'valid/l2': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1<save_to_csv>
def objective_with_prune(trial: Trial, fast_check=True, target_meter=0):

 folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
valid_score = 0
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm_with_pruning(trial, train_data, valid_data, cat_features=category_cols,
num_rounds=1000)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
valid_score += log["valid/l2"]
if fast_check:
break
valid_score /= len(models0)
 return valid_score
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission.csv", index=False )<init_hyperparams>
study = optuna.create_study(pruner=optuna.pruners.MedianPruner(n_warmup_steps=5))
study.optimize(objective_with_prune, n_trials=50 )
Dropout_model = 0.25
FVC_weight = 0.15
Confidence_weight = 0.15<install_modules>
print('Best trial: score {}, params {}'.format(study.best_trial.value, study.best_trial.params))
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index<set_options>
trials_df = study.trials_dataframe()
trials_df
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 seed_everything(45 )<set_options>
study = optuna.create_study(pruner=optuna.pruners.SuccessiveHalvingPruner(min_resource=2, reduction_factor=4, min_early_stopping_rate=1))
study.optimize(objective_with_prune, timeout=timeout )
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<load_from_csv>
print(f'Executed {len(study.trials)} trials, best score {study.best_value} with best_params {study.best_params}' )
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )<concatenate>
trials_df = study.trials_dataframe()
trials_df.to_csv('trials_history.csv' )
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )<feature_engineering>
print("Training model with best_params {}".format(study.best_params))
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
try:
del train_df
del train_df_original
del weather_train_df
except:
pass
gc.collect()
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 print('preprocessing weather...')
 weather_test_df = timestamp_align(weather_test_df)
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 weather_test_df = weather_test_df[['site_id', 'timestamp'] + weather_cols]
reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )<split>
del df_2d
del site_ids_offsets
gc.collect()
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.95 )<compute_test_metric>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left', copy=False)
 target_test_df = pd.merge(target_test_df, building_stats_df, on=['building_id', 'meter'], how='left', copy=False)
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left', copy=False)
 X_test = target_test_df[feature_cols + category_cols]
return X_test
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
%%time
X_test = create_X(test_df, target_meter=0)
 gc.collect()
y_test0 = pred(X_test, models0)
 del X_test
gc.collect()
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
%%time
X_test = create_X(test_df, target_meter=1)
 gc.collect()
y_test1 = pred(X_test, models1)
 del X_test
gc.collect()
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
%%time
X_test = create_X(test_df, target_meter=2)
 gc.collect()
y_test2 = pred(X_test, models2)
 del X_test
gc.collect()
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
%%time
X_test = create_X(test_df, target_meter=3)
 gc.collect()
y_test3 = pred(X_test, models3)
 del X_test
gc.collect()
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
print(np.sum(y_test0 < 0))
print(np.sum(y_test1 < 0))
print(np.sum(y_test2 < 0))
print(np.sum(y_test3 < 0))
y_test0 = np.where(y_test0 < 0, 0, y_test0)
 y_test1 = np.where(y_test1 < 0, 0, y_test1)
 y_test2 = np.where(y_test2 < 0, 0, y_test2)
 y_test3 = np.where(y_test3 < 0, 0, y_test3 )
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )<feature_engineering>
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )